@Test
public void testAverageIntDataElement() {
DataElement dataElement = createDataElement('A', categoryCombo);
dataElement.setType(DataElement.VALUE_TYPE_INT);
dataElement.setAggregationOperator(DataElement.AGGREGATION_OPERATOR_AVERAGE_SUM);
dataElementIds.add(dataElementService.addDataElement(dataElement));
DataSet dataSet = createDataSet('A', new YearlyPeriodType());
dataSet.getDataElements().add(dataElement);
dataSetService.addDataSet(dataSet);
dataElement.getDataSets().add(dataSet);
dataElementService.updateDataElement(dataElement);
Period dataPeriod =
createPeriod(new YearlyPeriodType(), getDate(2000, 1, 1), getDate(2000, 12, 31));
OrganisationUnit unit = createOrganisationUnit('A');
organisationUnitIds.add(organisationUnitService.addOrganisationUnit(unit));
dataValueService.addDataValue(
createDataValue(dataElement, dataPeriod, unit, "100", categoryOptionCombo));
Period periodA =
createPeriod(new MonthlyPeriodType(), getDate(2000, 3, 1), getDate(2000, 3, 31));
Period periodB =
createPeriod(new MonthlyPeriodType(), getDate(2000, 4, 1), getDate(2000, 4, 30));
Period periodC =
createPeriod(new QuarterlyPeriodType(), getDate(2000, 3, 1), getDate(2000, 5, 31));
periodIds.add(periodService.addPeriod(periodA));
periodIds.add(periodService.addPeriod(periodB));
periodIds.add(periodService.addPeriod(periodC));
dataMartEngine.export(dataElementIds, indicatorIds, periodIds, organisationUnitIds);
assertEquals(
100.0,
aggregatedDataValueService.getAggregatedValue(
dataElement, categoryOptionCombo, periodA, unit),
DELTA);
assertEquals(
100.0,
aggregatedDataValueService.getAggregatedValue(
dataElement, categoryOptionCombo, periodB, unit),
DELTA);
assertEquals(
100.0,
aggregatedDataValueService.getAggregatedValue(
dataElement, categoryOptionCombo, periodC, unit),
DELTA);
}
public String execute() throws Exception {
// ---------------------------------------------------------------------
// Prepare values
// ---------------------------------------------------------------------
code = nullIfEmpty(code);
shortName = nullIfEmpty(shortName);
description = nullIfEmpty(description);
PeriodType periodType = PeriodType.getPeriodTypeByName(frequencySelect);
DataSet dataSet = new DataSet(name, shortName, code, periodType);
MapLegendSet legendSet = mappingService.getMapLegendSet(selectedLegendSetId);
dataSet.setExpiryDays(expiryDays);
dataSet.setTimelyDays(timelyDays);
dataSet.setSkipAggregation(skipAggregation);
for (String id : dataElementsSelectedList) {
dataSet.addDataElement(dataElementService.getDataElement(Integer.parseInt(id)));
}
Set<Indicator> indicators = new HashSet<Indicator>();
for (String id : indicatorsSelectedList) {
indicators.add(indicatorService.getIndicator(Integer.parseInt(id)));
}
if (categoryComboId != null) {
dataSet.setCategoryCombo(categoryService.getDataElementCategoryCombo(categoryComboId));
}
dataSet.setDescription(description);
dataSet.setVersion(1);
dataSet.setMobile(false);
dataSet.setIndicators(indicators);
dataSet.setNotificationRecipients(userGroupService.getUserGroup(notificationRecipients));
dataSet.setAllowFuturePeriods(allowFuturePeriods);
dataSet.setFieldCombinationRequired(fieldCombinationRequired);
dataSet.setValidCompleteOnly(validCompleteOnly);
dataSet.setNotifyCompletingUser(notifyCompletingUser);
dataSet.setApproveData(approveData);
dataSet.setSkipOffline(skipOffline);
dataSet.setDataElementDecoration(dataElementDecoration);
dataSet.setRenderAsTabs(renderAsTabs);
dataSet.setRenderHorizontally(renderHorizontally);
dataSet.setLegendSet(legendSet);
dataSetService.addDataSet(dataSet);
userService.assignDataSetToUserRole(dataSet);
return SUCCESS;
}
@Override
public String execute() {
if (dataElementGroupId > 0) {
DataElementGroup group =
dataElementService.getDataElementGroup(dataElementGroupId.intValue());
dataElements = new ArrayList<>(group.getMembers());
} else {
dataElements = new ArrayList<>(dataElementService.getAllDataElements());
}
Collections.sort(dataElements, new IdentifiableObjectNameComparator());
for (DataElement dataElement : dataElements) {
DataElementCategoryCombo categoryCombo = dataElement.getCategoryCombo();
Set<DataElementCategoryOptionCombo> optionCombos = categoryCombo.getOptionCombos();
if (optionCombos.size() > 1) {
for (DataElementCategoryOptionCombo optionCombo : optionCombos) {
DataElementOperand operand =
new DataElementOperand(
dataElement.getUid(),
optionCombo.getUid(),
dataElement.getName() + optionCombo.getName());
operands.add(operand);
}
} else {
DataElementOperand operand =
new DataElementOperand(
dataElement.getUid(),
optionCombos.iterator().next().getUid(),
dataElement.getName());
operands.add(operand);
}
}
return SUCCESS;
}
@Override
public void setUpTest() {
organisationUnit = createOrganisationUnit('A');
organisationUnitLevel = new OrganisationUnitLevel(1, "Level");
organisationUnitService.addOrganisationUnit(organisationUnit);
organisationUnitService.addOrganisationUnitLevel(organisationUnitLevel);
indicatorGroup = createIndicatorGroup('A');
indicatorService.addIndicatorGroup(indicatorGroup);
indicatorType = createIndicatorType('A');
indicatorService.addIndicatorType(indicatorType);
indicator = createIndicator('A', indicatorType);
indicatorService.addIndicator(indicator);
dataElement = createDataElement('A');
dataElementService.addDataElement(dataElement);
dataElementGroup = createDataElementGroup('A');
dataElementGroup.getMembers().add(dataElement);
dataElementService.addDataElementGroup(dataElementGroup);
periodType = periodService.getPeriodTypeByName(MonthlyPeriodType.NAME);
period = createPeriod(periodType, getDate(2000, 1, 1), getDate(2000, 2, 1));
periodService.addPeriod(period);
mapLegendSet = createLegendSet('A');
legendService.addLegendSet(mapLegendSet);
internalMapLayer = new InternalMapLayer();
internalMapLayer.setRadiusLow(15);
internalMapLayer.setRadiusHigh(35);
internalMapLayer.setColorLow(Color.YELLOW);
internalMapLayer.setColorHigh(Color.RED);
internalMapLayer.setOpacity(0.5f);
}
@Override
public void setUpTest() {
organisationUnit = createOrganisationUnit('A');
organisationUnitService.addOrganisationUnit(organisationUnit);
Program program = createProgram('A', new HashSet<ProgramStage>(), organisationUnit);
programService.addProgram(program);
stageA = new ProgramStage("A", program);
programStageService.saveProgramStage(stageA);
DataElement dataElementA = createDataElement('A');
DataElement dataElementB = createDataElement('B');
dataElementService.addDataElement(dataElementA);
dataElementService.addDataElement(dataElementB);
ProgramStageDataElement stageDeA = new ProgramStageDataElement(stageA, dataElementA, false, 1);
ProgramStageDataElement stageDeB = new ProgramStageDataElement(stageA, dataElementB, false, 2);
programStageDataElementService.addProgramStageDataElement(stageDeA);
programStageDataElementService.addProgramStageDataElement(stageDeB);
List<ProgramStageDataElement> psDataElements = new ArrayList<>();
psDataElements.add(stageDeA);
psDataElements.add(stageDeB);
stageB = new ProgramStage("B", program);
programStageService.saveProgramStage(stageB);
Set<ProgramStage> programStages = new HashSet<>();
programStages.add(stageA);
programStages.add(stageB);
program.setProgramStages(programStages);
programService.updateProgram(program);
sectionA = new ProgramStageSection();
sectionA.setName("A");
sectionA.setProgramStageDataElements(psDataElements);
sectionA.setSortOrder(1);
sectionB = new ProgramStageSection();
sectionB.setName("B");
sectionB.setSortOrder(2);
Set<ProgramStageSection> sections = new HashSet<>();
sections.add(sectionA);
sections.add(sectionB);
stageA.setProgramStageSections(sections);
}
public String execute() {
// ---------------------------------------------------------------------
// Get group members
// ---------------------------------------------------------------------
if (id != null) {
DataElementGroup group = dataElementService.getDataElementGroup(id);
groupMembers = new ArrayList<DataElement>(group.getMembers());
Collections.sort(groupMembers, new IdentifiableObjectNameComparator());
}
return SUCCESS;
}
protected void addDataValue(
OrganisationUnit unit,
Period period,
String expression,
String value,
Set<DataValue> oldList,
Set<DataValue> newList) {
// value = value.replaceAll( "\\.", "" ).replace( ",", "." );
DataElementOperand operand =
expressionService.getOperandsInExpression(expression).iterator().next();
DataElement dataElement = dataElementService.getDataElement(operand.getDataElementId());
DataElementCategoryOptionCombo optionCombo =
categoryService.getDataElementCategoryOptionCombo(operand.getOptionComboId());
String storedBy = currentUserService.getCurrentUsername();
DataValue dataValue = dataValueService.getDataValue(unit, dataElement, period, optionCombo);
if (dataValue == null) {
dataValue =
new DataValue(dataElement, period, unit, value, storedBy, new Date(), null, optionCombo);
dataValueService.addDataValue(dataValue);
newList.add(dataValue);
} else {
DataValue backedUpDataValue =
new DataValue(dataElement, period, unit, dataValue.getValue(), optionCombo);
oldList.add(backedUpDataValue);
dataValue.setValue(value);
dataValue.setTimestamp(new Date());
dataValue.setStoredBy(storedBy);
dataValueService.updateDataValue(dataValue);
}
}
public String execute() {
DataElement dataElement = dataElementService.getDataElement(dataElementId);
DataElementCategoryOptionCombo categoryOptionCombo =
categoryService.getDataElementCategoryOptionCombo(categoryOptionComboId);
Period period = PeriodType.createPeriodExternalId(periodId);
OrganisationUnit source = organisationUnitService.getOrganisationUnit(organisationUnitId);
DataValue dataValue =
dataValueService.getDataValue(source, dataElement, period, categoryOptionCombo);
boolean isMarked = dataValue.isFollowup();
dataValue.setFollowup(!isMarked);
dataValueService.updateDataValue(dataValue);
message = !isMarked ? "marked" : "unmarked";
log.info(!isMarked ? "Data value marked for follow-up" : "Data value unmarked for follow-up");
return SUCCESS;
}
private void applyAggregationLevels(List<AnalyticsTable> tables) {
int maxLevels = organisationUnitService.getNumberOfOrganisationalLevels();
boolean hasAggLevels = false;
levelLoop:
for (int i = 0; i < maxLevels; i++) {
int level = maxLevels - i;
Collection<String> dataElements =
IdentifiableObjectUtils.getUids(
dataElementService.getDataElementsByAggregationLevel(level));
if (dataElements.isEmpty()) {
continue levelLoop;
}
hasAggLevels = true;
ConcurrentLinkedQueue<AnalyticsTable> tableQ = new ConcurrentLinkedQueue<>(tables);
List<Future<?>> futures = new ArrayList<>();
for (int j = 0; j < getProcessNo(); j++) {
futures.add(tableManager.applyAggregationLevels(tableQ, dataElements, level));
}
ConcurrentUtils.waitForCompletion(futures);
}
if (hasAggLevels) {
vacuumTables(tables);
log.info("Vacuumed tables");
}
}
@Override
public String execute() throws Exception {
ExportParams params = new ExportParams();
if (dataValue || dataValueDaily) {
params.setCategories(null);
params.setCategoryCombos(null);
params.setCategoryOptions(null);
params.setCategoryOptionCombos(null);
/*
* params.setDataElementGroups( null );
* params.setDataElementGroupSets( null ); params.setIndicators(
* null ); params.setIndicatorTypes( null );
* params.setIndicatorGroups( null ); params.setIndicatorGroupSets(
* null ); params.setDataDictionaries( null ); params.setDataSets(
* null ); params.setOrganisationUnitGroups( null );
* params.setOrganisationUnitGroupSets( null );
* params.setOrganisationUnitLevels( null );
* params.setValidationRules( null ); params.setReports( null );
* params.setReportTables( null ); params.setPeriods( null ); //
* TODO Include only relevant periods params.setCharts( null );
* params.setPeriods( null );
*/
Set<Integer> dataElementz = new HashSet<>();
Collection<DataElement> children = dataElementService.getAllDataElements();
for (DataElement child : children) {
dataElementz.add(child.getId());
}
params.setDataElements(dataElementz);
Set<Integer> orgUnits = new HashSet<>();
Collection<OrganisationUnit> orgChildren = organisationUnitService.getAllOrganisationUnits();
for (OrganisationUnit child : orgChildren) {
orgUnits.add(child.getId());
}
params.setOrganisationUnits(orgUnits);
}
if (dataValue) {
params.setMetaData(false);
}
if (dataValueDaily) {
params.setMetaData(false);
}
// params.setMetaData( true );
params.setIncludeDataValues(true);
params.setCurrentUser(currentUserService.getCurrentUser());
params.setDataValue(dataValue);
params.setDataValueDaily(dataValueDaily);
params.setI18n(i18n);
params.setFormat(format);
params.setStartDate(getMediumDate(startDate));
params.setEndDate(getMediumDate(endDate));
ExportService exportService = serviceProvider.provide(exportFormat);
inputStream = exportService.exportData(params);
fileName = FILENAME;
return SUCCESS;
}
@RequestMapping(method = RequestMethod.POST, produces = "text/plain")
public void saveDataValue(
@RequestParam String de,
@RequestParam String cc,
@RequestParam String pe,
@RequestParam String ou,
@RequestParam String value,
HttpServletResponse response) {
DataElement dataElement = dataElementService.getDataElement(de);
if (dataElement == null) {
ContextUtils.conflictResponse(response, "Illegal data element identifier: " + de);
return;
}
DataElementCategoryOptionCombo categoryOptionCombo =
categoryService.getDataElementCategoryOptionCombo(cc);
if (categoryOptionCombo == null) {
ContextUtils.conflictResponse(response, "Illegal category option combo identifier: " + cc);
return;
}
Period period = PeriodType.getPeriodFromIsoString(pe);
if (period == null) {
ContextUtils.conflictResponse(response, "Illegal period identifier: " + pe);
return;
}
OrganisationUnit organisationUnit = organisationUnitService.getOrganisationUnit(ou);
if (organisationUnit == null) {
ContextUtils.conflictResponse(response, "Illegal organisation unit identifier: " + ou);
return;
}
if (dataSetService.isLocked(dataElement, period, organisationUnit, null)) {
ContextUtils.conflictResponse(response, "Data set is locked");
return;
}
value = StringUtils.trimToNull(value);
String storedBy = currentUserService.getCurrentUsername();
Date now = new Date();
DataValue dataValue =
dataValueService.getDataValue(organisationUnit, dataElement, period, categoryOptionCombo);
if (dataValue == null) {
if (value != null) {
dataValue =
new DataValue(
dataElement,
period,
organisationUnit,
value,
storedBy,
now,
null,
categoryOptionCombo);
dataValueService.addDataValue(dataValue);
}
} else {
dataValue.setValue(value);
dataValue.setTimestamp(now);
dataValue.setStoredBy(storedBy);
dataValueService.updateDataValue(dataValue);
}
}
@Override
public void setUpTest() {
mockDataValueBatchHandler = new MockBatchHandler<>();
mockBatchHandlerFactory = new MockBatchHandlerFactory();
mockBatchHandlerFactory.registerBatchHandler(
DataValueBatchHandler.class, mockDataValueBatchHandler);
setDependency(dataValueSetService, "batchHandlerFactory", mockBatchHandlerFactory);
categoryOptionA = createCategoryOption('A');
categoryOptionB = createCategoryOption('B');
categoryA = createDataElementCategory('A', categoryOptionA, categoryOptionB);
categoryComboA = createCategoryCombo('A', categoryA);
ocDef = categoryService.getDefaultDataElementCategoryOptionCombo();
ocA = createCategoryOptionCombo(categoryComboA, categoryOptionA);
ocB = createCategoryOptionCombo(categoryComboA, categoryOptionB);
deA = createDataElement('A');
deB = createDataElement('B');
deC = createDataElement('C');
deD = createDataElement('D');
dsA = createDataSet('A', new MonthlyPeriodType());
ouA = createOrganisationUnit('A');
ouB = createOrganisationUnit('B');
ouC = createOrganisationUnit('C');
peA =
createPeriod(
PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME),
getDate(2012, 1, 1),
getDate(2012, 1, 31));
peB =
createPeriod(
PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME),
getDate(2012, 2, 1),
getDate(2012, 2, 29));
ocA.setUid("kjuiHgy67hg");
ocB.setUid("Gad33qy67g5");
deA.setUid("f7n9E0hX8qk");
deB.setUid("Ix2HsbDMLea");
deC.setUid("eY5ehpbEsB7");
dsA.setUid("pBOMPrpg1QX");
ouA.setUid("DiszpKrYNg8");
ouB.setUid("BdfsJfj87js");
ouC.setUid("j7Hg26FpoIa");
ocA.setCode("OC_A");
ocB.setCode("OC_B");
deA.setCode("DE_A");
deB.setCode("DE_B");
deC.setCode("DE_C");
deD.setCode("DE_D");
dsA.setCode("DS_A");
ouA.setCode("OU_A");
ouB.setCode("OU_B");
ouC.setCode("OU_C");
categoryService.addDataElementCategoryOption(categoryOptionA);
categoryService.addDataElementCategoryOption(categoryOptionB);
categoryService.addDataElementCategory(categoryA);
categoryService.addDataElementCategoryCombo(categoryComboA);
categoryService.addDataElementCategoryOptionCombo(ocA);
categoryService.addDataElementCategoryOptionCombo(ocB);
dataElementService.addDataElement(deA);
dataElementService.addDataElement(deB);
dataElementService.addDataElement(deC);
dataElementService.addDataElement(deD);
dataSetService.addDataSet(dsA);
organisationUnitService.addOrganisationUnit(ouA);
organisationUnitService.addOrganisationUnit(ouB);
organisationUnitService.addOrganisationUnit(ouC);
periodService.addPeriod(peA);
periodService.addPeriod(peB);
user = createUser('A');
user.setOrganisationUnits(Sets.newHashSet(ouA, ouB));
CurrentUserService currentUserService = new MockCurrentUserService(user);
setDependency(dataValueSetService, "currentUserService", currentUserService);
}
@Override
public String execute() throws Exception {
for (String aggregateValue : aggregateValues) {
// -----------------------------------------------------------------
// Get params
// -----------------------------------------------------------------
String[] info = aggregateValue.split(SEPERATE_SIGN);
int dataElementId = Integer.parseInt(info[0]);
int optionComboId = Integer.parseInt(info[1]);
String periodIsoId = info[2];
int orgunitId = Integer.parseInt(info[3]);
String resultValue = info[4];
// -----------------------------------------------------------------
// Create objects
// -----------------------------------------------------------------
DataElement dataElement = dataElementService.getDataElement(dataElementId);
DataElementCategoryOptionCombo optionCombo =
categoryService.getDataElementCategoryOptionCombo(optionComboId);
DataElementCategoryOptionCombo attributeOptioncombo =
categoryService.getDefaultDataElementCategoryOptionCombo();
Period period = PeriodType.getPeriodFromIsoString(periodIsoId);
OrganisationUnit orgunit = organisationUnitService.getOrganisationUnit(orgunitId);
DataValue dataValue =
dataValueService.getDataValue(
dataElement, period, orgunit, optionCombo, attributeOptioncombo);
// -----------------------------------------------------------------
// Save/Update/Delete data-values
// -----------------------------------------------------------------
if (resultValue != "0.0") {
if (dataValue == null) {
dataValue =
new DataValue(
dataElement,
period,
orgunit,
optionCombo,
attributeOptioncombo,
"" + resultValue,
CaseAggregationCondition.AUTO_STORED_BY,
new Date(),
null);
dataValueService.addDataValue(dataValue);
} else {
dataValue.setValue(resultValue);
dataValue.setLastUpdated(new Date());
dataValue.setStoredBy(CaseAggregationCondition.AUTO_STORED_BY);
dataValueService.updateDataValue(dataValue);
}
} else if (dataValue != null) {
dataValueService.deleteDataValue(dataValue);
}
}
return SUCCESS;
}
@Transactional
public void export(
Collection<Integer> dataElementIds,
Collection<Integer> indicatorIds,
Collection<Integer> periodIds,
Collection<Integer> organisationUnitIds,
Collection<Integer> organisationUnitGroupIds,
TaskId id) {
final int cpuCores = SystemUtils.getCpuCores();
Clock clock =
new Clock()
.startClock()
.logTime(
"Data mart export process started, number of CPU cores: "
+ cpuCores
+ ", "
+ SystemUtils.getMemoryString());
notifier.clear(id, DATAMART).notify(id, DATAMART, "Data mart export process started");
// ---------------------------------------------------------------------
// Recreate temporary tables
// ---------------------------------------------------------------------
dataMartManager.dropTempAggregatedTables();
dataMartManager.createTempAggregatedTables();
clock.logTime("Recreated temporary tables");
// ---------------------------------------------------------------------
// Replace null with empty collection
// ---------------------------------------------------------------------
dataElementIds = dataElementIds != null ? dataElementIds : new ArrayList<Integer>();
indicatorIds = indicatorIds != null ? indicatorIds : new ArrayList<Integer>();
periodIds = periodIds != null ? periodIds : new ArrayList<Integer>();
organisationUnitIds =
organisationUnitIds != null ? organisationUnitIds : new ArrayList<Integer>();
organisationUnitGroupIds =
organisationUnitGroupIds != null ? organisationUnitGroupIds : new ArrayList<Integer>();
clock.logTime(
"Data elements: "
+ dataElementIds.size()
+ ", indicators: "
+ indicatorIds.size()
+ ", periods: "
+ periodIds.size()
+ ", org units: "
+ organisationUnitIds.size());
// ---------------------------------------------------------------------
// Get objects
// ---------------------------------------------------------------------
final Collection<Indicator> indicators = indicatorService.getIndicators(indicatorIds);
final Collection<Period> periods = periodService.getPeriods(periodIds);
final List<OrganisationUnit> organisationUnits =
new ArrayList<OrganisationUnit>(
organisationUnitService.getOrganisationUnits(organisationUnitIds));
final Collection<OrganisationUnitGroup> organisationUnitGroups =
organisationUnitGroupService.getOrganisationUnitGroups(organisationUnitGroupIds);
final Collection<DataElement> dataElements = dataElementService.getDataElements(dataElementIds);
clock.logTime("Retrieved meta-data objects");
notifier.notify(id, DATAMART, "Filtering meta-data");
// ---------------------------------------------------------------------
// Filter objects
// ---------------------------------------------------------------------
organisationUnitService.filterOrganisationUnitsWithoutData(organisationUnits);
FilterUtils.filter(dataElements, new AggregatableDataElementFilter());
FilterUtils.filter(dataElements, new DataElementWithAggregationFilter());
expressionService.filterInvalidIndicators(indicators);
clock.logTime("Filtered objects");
notifier.notify(id, DATAMART, "Loading indicators");
// ---------------------------------------------------------------------
// Explode indicator expressions
// ---------------------------------------------------------------------
for (Indicator indicator : indicators) {
indicator.setExplodedNumerator(expressionService.explodeExpression(indicator.getNumerator()));
indicator.setExplodedDenominator(
expressionService.explodeExpression(indicator.getDenominator()));
}
clock.logTime("Exploded indicator expressions");
notifier.notify(id, DATAMART, "Loading data elements");
// ---------------------------------------------------------------------
// Get operands
// ---------------------------------------------------------------------
final Collection<DataElementOperand> dataElementOperands =
categoryService.getOperands(dataElements);
final List<DataElementOperand> indicatorOperands =
new ArrayList<DataElementOperand>(
categoryService.populateOperands(
expressionService.getOperandsInIndicators(indicators)));
Set<DataElementOperand> allOperands = new HashSet<DataElementOperand>();
allOperands.addAll(dataElementOperands);
allOperands.addAll(indicatorOperands);
clock.logTime("Retrieved operands: " + allOperands.size());
notifier.notify(id, DATAMART, "Loading periods");
// ---------------------------------------------------------------------
// Filter out future periods
// ---------------------------------------------------------------------
FilterUtils.filter(periods, new PastAndCurrentPeriodFilter());
clock.logTime("Number of periods: " + periods.size());
notifier.notify(id, DATAMART, "Filtering data elements without data");
// ---------------------------------------------------------------------
// Remove operands without data
// ---------------------------------------------------------------------
allOperands = dataMartManager.getOperandsWithData(allOperands);
indicatorOperands.retainAll(allOperands);
clock.logTime(
"Number of operands with data: "
+ allOperands.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Populating crosstabulation table");
// ---------------------------------------------------------------------
// Create crosstabtable
// ---------------------------------------------------------------------
final Collection<Integer> intersectingPeriodIds =
ConversionUtils.getIdentifiers(Period.class, periodService.getIntersectionPeriods(periods));
final PeriodHierarchy periodHierarchy = periodService.getPeriodHierarchy(periods);
final Set<Integer> orgUnitChildrenIds =
organisationUnitService.getOrganisationUnitHierarchy().getChildren(organisationUnitIds);
final List<Integer> crossTabOrgUnitIds = new ArrayList<Integer>(orgUnitChildrenIds);
final String key = crossTabService.createCrossTabTable(crossTabOrgUnitIds);
final List<DataElementOperand> operandList = new ArrayList<DataElementOperand>(allOperands);
Collections.shuffle(operandList);
final List<List<DataElementOperand>> operandPages =
new PaginatedList<DataElementOperand>(operandList).setNumberOfPages(cpuCores).getPages();
List<Future<?>> crossTabFutures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
crossTabFutures.add(
crossTabService.populateCrossTabTable(
operandPage, intersectingPeriodIds, crossTabOrgUnitIds, key));
}
ConcurrentUtils.waitForCompletion(crossTabFutures);
clock.logTime("Populated crosstab table, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Exporting data element data");
final boolean isDataElements = true;
final boolean isIndicators = indicators != null && indicators.size() > 0;
final int groupLevel =
(Integer)
systemSettingManager.getSystemSetting(
KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL);
final boolean isGroups =
organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0;
if (isDataElements) {
// -----------------------------------------------------------------
// 1. Export data element values
// -----------------------------------------------------------------
if (operandList.size() > 0) {
final OrganisationUnitHierarchy orgUnitHierarchy =
organisationUnitService
.getOrganisationUnitHierarchy()
.prepareChildren(organisationUnits);
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
futures.add(
dataElementDataMart.exportDataValues(
operandPage,
periods,
organisationUnits,
null,
periodHierarchy,
orgUnitHierarchy,
AggregatedDataValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
}
clock.logTime(
"Exported values for data element operands ("
+ operandList.size()
+ "), pages: "
+ operandPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping data element index");
// -----------------------------------------------------------------
// 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropDataValueIndex();
clock.logTime("Dropped data element index");
notifier.notify(id, DATAMART, "Deleting existing data element data");
// -----------------------------------------------------------------
// 3. Delete existing aggregated data values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedDataValues(periodIds);
clock.logTime("Deleted existing data element data");
notifier.notify(id, DATAMART, "Copying data element data from temporary table");
// -----------------------------------------------------------------
// 4. Copy aggregated data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedDataValuesFromTemp();
clock.logTime("Copied data element data from temporary table");
notifier.notify(id, DATAMART, "Creating data element index");
// -----------------------------------------------------------------
// 5. Create data element index
// -----------------------------------------------------------------
dataMartManager.createDataValueIndex();
clock.logTime("Created data element index");
}
if (isGroups && isDataElements) {
// -----------------------------------------------------------------
// 1. Export data element values
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Exporting data element org unit data");
Collection<OrganisationUnit> groupOrganisationUnits =
new HashSet<OrganisationUnit>(organisationUnits);
FilterUtils.filter(
groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter(groupLevel));
if (operandList.size() > 0) {
final OrganisationUnitHierarchy orgUnitHierarchy =
organisationUnitService
.getOrganisationUnitHierarchy()
.prepareChildren(groupOrganisationUnits, organisationUnitGroups);
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
futures.add(
dataElementDataMart.exportDataValues(
operandPage,
periods,
groupOrganisationUnits,
organisationUnitGroups,
periodHierarchy,
orgUnitHierarchy,
AggregatedOrgUnitDataValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
}
clock.logTime(
"Exported values for data element operands ("
+ operandList.size()
+ "), pages: "
+ operandPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping data element data indexes");
// -----------------------------------------------------------------
// 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropOrgUnitDataValueIndex();
clock.logTime("Dropped org unit data element index");
notifier.notify(id, DATAMART, "Deleting existing org unit data element data");
// -----------------------------------------------------------------
// 3. Delete existing aggregated data values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitDataValues(periodIds);
clock.logTime("Deleted existing aggregated org unit datavalues");
notifier.notify(id, DATAMART, "Copying org unit data element data");
// -----------------------------------------------------------------
// 4. Copy aggregated org unit data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp();
clock.logTime("Copied org unit data element data from temporary table");
notifier.notify(id, DATAMART, "Creating org unit data element index");
// -----------------------------------------------------------------
// 5. Create org unit data element index
// -----------------------------------------------------------------
dataMartManager.createOrgUnitDataValueIndex();
clock.logTime("Created org unit data element index");
notifier.notify(id, DATAMART, "Exporting data for org unit indicator data");
}
crossTabService.dropCrossTabTable(key);
List<List<OrganisationUnit>> organisationUnitPages =
new PaginatedList<OrganisationUnit>(organisationUnits)
.setNumberOfPages(cpuCores)
.getPages();
if (isIndicators) {
// -----------------------------------------------------------------
// 1. Create and populate aggregated data cache
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Populating aggregated data cache");
crossTabService.createAggregatedDataCache(indicatorOperands, key);
List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
aggregatedDataCacheFutures.add(
crossTabService.populateAggregatedDataCache(
indicatorOperands, periods, organisationUnitPage, key));
}
ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);
clock.logTime(
"Created aggregated data cache, number of indicator operands: "
+ indicatorOperands.size());
notifier.notify(id, DATAMART, "Exporting indicator data");
// -----------------------------------------------------------------
// 2. Export indicator values
// -----------------------------------------------------------------
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
futures.add(
indicatorDataMart.exportIndicatorValues(
indicators,
periods,
organisationUnitPage,
null,
indicatorOperands,
AggregatedIndicatorValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
clock.logTime(
"Exported values for indicators ("
+ indicators.size()
+ "), pages: "
+ organisationUnitPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping indicator index");
// -----------------------------------------------------------------
// 3. Drop aggregated data cache and indicator index
// -----------------------------------------------------------------
crossTabService.dropAggregatedDataCache(key);
dataMartManager.dropIndicatorValueIndex();
clock.logTime("Dropped indicator index, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Deleting existing indicator data");
// -----------------------------------------------------------------
// 4. Delete existing aggregated indicator values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedIndicatorValues(periodIds);
clock.logTime("Deleted existing indicator data");
notifier.notify(id, DATAMART, "Copying indicator data from temporary table");
// -----------------------------------------------------------------
// 5. Copy aggregated data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedIndicatorValuesFromTemp();
clock.logTime("Copied indicator data from temporary table");
notifier.notify(id, DATAMART, "Creating indicator index");
// -----------------------------------------------------------------
// 6. Create indicator index
// -----------------------------------------------------------------
dataMartManager.createIndicatorValueIndex();
clock.logTime("Created indicator index");
}
if (isGroups && isIndicators) {
// -----------------------------------------------------------------
// 1. Create aggregated data cache
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Populating aggregated data cache");
crossTabService.createAggregatedOrgUnitDataCache(indicatorOperands, key);
List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
aggregatedDataCacheFutures.add(
crossTabService.populateAggregatedOrgUnitDataCache(
indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key));
}
ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);
clock.logTime("Created aggregated org unit data cache");
notifier.notify(id, DATAMART, "Exporting org unit indicator data");
// -----------------------------------------------------------------
// 2. Export indicator values
// -----------------------------------------------------------------
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
futures.add(
indicatorDataMart.exportIndicatorValues(
indicators,
periods,
organisationUnitPage,
organisationUnitGroups,
indicatorOperands,
AggregatedOrgUnitIndicatorValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
clock.logTime(
"Exported values for indicators ("
+ indicators.size()
+ "), pages: "
+ organisationUnitPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping org unit indicator index");
// -----------------------------------------------------------------
// 3. Drop aggregated data cache and indicator index
// -----------------------------------------------------------------
crossTabService.dropAggregatedOrgUnitDataCache(key);
dataMartManager.dropOrgUnitIndicatorValueIndex();
clock.logTime("Dropped org unit indicator index, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Deleting existing org unit indicator data");
// -----------------------------------------------------------------
// 4. Delete existing aggregated indicator values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitIndicatorValues(periodIds);
clock.logTime("Deleted existing aggregated org unit indicatorvalues");
notifier.notify(id, DATAMART, "Copying org unit indicator data from temporary table");
// -----------------------------------------------------------------
// 5. Copy aggregated org unit indicator values from temp table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();
clock.logTime("Copied org unit indicator data from temporary table");
notifier.notify(id, DATAMART, "Creating org unit indicator indexes");
// -----------------------------------------------------------------
// 6. Create org unit indicator index
// -----------------------------------------------------------------
dataMartManager.createOrgUnitIndicatorValueIndex();
clock.logTime("Created org unit indicator index");
}
// ---------------------------------------------------------------------
// Drop temporary tables
// ---------------------------------------------------------------------
dataMartManager.dropTempAggregatedTables();
clock.logTime("Dropped crosstab table");
clock.logTime("Data mart export process completed");
notifier.notify(id, DATAMART, INFO, "Data mart process completed", true);
}
@Transactional
public void matchObject(int importObjectId, int existingObjectId) {
ImportObject importObject = importObjectStore.getImportObject(importObjectId);
Object object = importObject.getObject();
// ---------------------------------------------------------------------
// Updates the name of the import object to the name of the existing
// object.
// ---------------------------------------------------------------------
if (object.getClass().equals(DataElement.class)) {
DataElement element = (DataElement) object;
element.setName(dataElementService.getDataElement(existingObjectId).getName());
} else if (object.getClass().equals(DataElementGroup.class)) {
DataElementGroup group = (DataElementGroup) object;
group.setName(dataElementService.getDataElementGroup(existingObjectId).getName());
} else if (object.getClass().equals(DataElementGroupSet.class)) {
DataElementGroupSet groupSet = (DataElementGroupSet) object;
groupSet.setName(dataElementService.getDataElementGroupSet(existingObjectId).getName());
} else if (object.getClass().equals(IndicatorType.class)) {
IndicatorType type = (IndicatorType) object;
type.setName(indicatorService.getIndicatorType(existingObjectId).getName());
} else if (object.getClass().equals(Indicator.class)) {
Indicator indicator = (Indicator) object;
indicator.setName(indicatorService.getIndicator(existingObjectId).getName());
} else if (object.getClass().equals(IndicatorGroup.class)) {
IndicatorGroup group = (IndicatorGroup) object;
group.setName(indicatorService.getIndicatorGroup(existingObjectId).getName());
} else if (object.getClass().equals(IndicatorGroupSet.class)) {
IndicatorGroupSet groupSet = (IndicatorGroupSet) object;
groupSet.setName(indicatorService.getIndicatorGroupSet(existingObjectId).getName());
} else if (object.getClass().equals(DataDictionary.class)) {
DataDictionary dictionary = (DataDictionary) object;
dictionary.setName(dataDictionaryService.getDataDictionary(existingObjectId).getName());
} else if (object.getClass().equals(DataSet.class)) {
DataSet dataSet = (DataSet) object;
dataSet.setName(dataSetService.getDataSet(existingObjectId).getName());
} else if (object.getClass().equals(OrganisationUnit.class)) {
OrganisationUnit unit = (OrganisationUnit) object;
unit.setName(organisationUnitService.getOrganisationUnit(existingObjectId).getName());
} else if (object.getClass().equals(OrganisationUnitGroup.class)) {
OrganisationUnitGroup group = (OrganisationUnitGroup) object;
group.setName(
organisationUnitGroupService.getOrganisationUnitGroup(existingObjectId).getName());
} else if (object.getClass().equals(OrganisationUnitGroupSet.class)) {
OrganisationUnitGroupSet groupSet = (OrganisationUnitGroupSet) object;
groupSet.setName(
organisationUnitGroupService.getOrganisationUnitGroupSet(existingObjectId).getName());
} else if (object.getClass().equals(OrganisationUnitLevel.class)) {
OrganisationUnitLevel level = (OrganisationUnitLevel) object;
level.setName(organisationUnitService.getOrganisationUnitLevel(existingObjectId).getName());
} else if (object.getClass().equals(ValidationRule.class)) {
ValidationRule validationRule = (ValidationRule) object;
validationRule.setName(validationRuleService.getValidationRule(existingObjectId).getName());
} else if (object.getClass().equals(Report.class)) {
Report report = (Report) object;
report.setName(reportService.getReport(existingObjectId).getName());
} else if (object.getClass().equals(ReportTable.class)) {
ReportTable reportTable = (ReportTable) object;
reportTable.setName(reportTableService.getReportTable(existingObjectId).getName());
} else if (object.getClass().equals(Chart.class)) {
Chart chart = (Chart) object;
chart.setName(chartService.getChart(existingObjectId).getName());
} else if (object.getClass().equals(DataValue.class)) {
DataValue dataValue = (DataValue) object;
dataValue =
updateDataValue(
dataValue,
dataValueService.getDataValue(
dataValue.getSource(),
dataValue.getDataElement(),
dataValue.getPeriod(),
dataValue.getOptionCombo()));
}
// ---------------------------------------------------------------------
// Sets the status of the import object to match, these objects will
// later be ignored on import all but is needed for matching of
// associations.
// ---------------------------------------------------------------------
importObject.setStatus(ImportObjectStatus.MATCH);
importObjectStore.updateImportObject(importObject);
}