private void deleteIndicatorsWithIndicatorType(int indicatorTypeId) {
Collection<ImportObject> importObjects = importObjectStore.getImportObjects(Indicator.class);
for (ImportObject importObject : importObjects) {
Indicator indicator = (Indicator) importObject.getObject();
if (indicator.getIndicatorType().getId() == indicatorTypeId) {
importObjectStore.deleteImportObject(importObject);
deleteMemberAssociations(GroupMemberType.INDICATORGROUP, indicator.getId());
deleteMemberAssociations(GroupMemberType.DATADICTIONARY_INDICATOR, indicator.getId());
}
}
}
private void deleteIndicatorsContainingDataElement(int dataElementId) {
Collection<ImportObject> importObjects = importObjectStore.getImportObjects(Indicator.class);
for (ImportObject importObject : importObjects) {
Indicator indicator = (Indicator) importObject.getObject();
if (containsIdentifier(indicator.getNumerator(), dataElementId)
|| containsIdentifier(indicator.getDenominator(), dataElementId)) {
importObjectStore.deleteImportObject(importObject);
deleteMemberAssociations(GroupMemberType.INDICATORGROUP, indicator.getId());
deleteMemberAssociations(GroupMemberType.DATADICTIONARY_INDICATOR, indicator.getId());
}
}
}
public void write(XMLWriter writer, ExportParams params) {
Collection<DataDictionary> dataDictionaries =
dataDictionaryService.getDataDictionaries(params.getDataDictionaries());
if (dataDictionaries != null && dataDictionaries.size() > 0) {
writer.openElement(COLLECTION_NAME);
for (DataDictionary dictionary : dataDictionaries) {
if (dictionary.getDataElements() != null) {
for (Indicator indicator : dictionary.getIndicators()) {
writer.openElement(ELEMENT_NAME);
writer.writeElement(FIELD_DATADICTIONARY, String.valueOf(dictionary.getId()));
writer.writeElement(FIELD_INDICATOR, String.valueOf(indicator.getId()));
writer.closeElement();
}
}
}
writer.closeElement();
}
}
@Override
public void setUpTest() {
// ---------------------------------------------------------------------
// Services
// ---------------------------------------------------------------------
dataIntegrityService = (DataIntegrityService) getBean(DataIntegrityService.ID);
dataElementService = (DataElementService) getBean(DataElementService.ID);
indicatorService = (IndicatorService) getBean(IndicatorService.ID);
dataSetService = (DataSetService) getBean(DataSetService.ID);
organisationUnitService = (OrganisationUnitService) getBean(OrganisationUnitService.ID);
organisationUnitGroupService =
(OrganisationUnitGroupService) getBean(OrganisationUnitGroupService.ID);
// ---------------------------------------------------------------------
// Objects
// ---------------------------------------------------------------------
elementA = createDataElement('A');
elementB = createDataElement('B');
elementC = createDataElement('C');
dataElementService.addDataElement(elementA);
dataElementService.addDataElement(elementB);
dataElementService.addDataElement(elementC);
indicatorTypeA = createIndicatorType('A');
indicatorService.addIndicatorType(indicatorTypeA);
indicatorA = createIndicator('A', indicatorTypeA);
indicatorB = createIndicator('B', indicatorTypeA);
indicatorC = createIndicator('C', indicatorTypeA);
indicatorA.setNumerator(" ");
indicatorB.setNumerator("Numerator");
indicatorB.setDenominator("Denominator");
indicatorC.setNumerator("Numerator");
indicatorC.setDenominator("Denominator");
indicatorService.addIndicator(indicatorA);
indicatorService.addIndicator(indicatorB);
indicatorService.addIndicator(indicatorC);
unitA = createOrganisationUnit('A');
unitB = createOrganisationUnit('B', unitA);
unitC = createOrganisationUnit('C', unitB);
unitD = createOrganisationUnit('D', unitC);
unitE = createOrganisationUnit('E', unitD);
unitF = createOrganisationUnit('F');
organisationUnitService.addOrganisationUnit(unitA);
organisationUnitService.addOrganisationUnit(unitB);
organisationUnitService.addOrganisationUnit(unitC);
organisationUnitService.addOrganisationUnit(unitD);
organisationUnitService.addOrganisationUnit(unitE);
organisationUnitService.addOrganisationUnit(unitF);
unitA.setParent(unitC);
organisationUnitService.updateOrganisationUnit(unitA);
dataSetA = createDataSet('A', new MonthlyPeriodType());
dataSetB = createDataSet('B', new QuarterlyPeriodType());
dataSetA.getDataElements().add(elementA);
dataSetA.getDataElements().add(elementB);
elementA.getDataSets().add(dataSetA);
elementB.getDataSets().add(dataSetA);
dataSetA.getSources().add(unitA);
unitA.getDataSets().add(dataSetA);
dataSetB.getDataElements().add(elementA);
elementA.getDataSets().add(dataSetB);
dataSetService.addDataSet(dataSetA);
dataSetService.addDataSet(dataSetB);
// ---------------------------------------------------------------------
// Groups
// ---------------------------------------------------------------------
elementGroupA = createDataElementGroup('A');
elementGroupA.getMembers().add(elementA);
elementA.getGroups().add(elementGroupA);
dataElementService.addDataElementGroup(elementGroupA);
indicatorGroupA = createIndicatorGroup('A');
indicatorGroupA.getMembers().add(indicatorA);
indicatorA.getGroups().add(indicatorGroupA);
indicatorService.addIndicatorGroup(indicatorGroupA);
unitGroupA = createOrganisationUnitGroup('A');
unitGroupB = createOrganisationUnitGroup('B');
unitGroupC = createOrganisationUnitGroup('C');
unitGroupD = createOrganisationUnitGroup('D');
unitGroupA.getMembers().add(unitA);
unitGroupA.getMembers().add(unitB);
unitGroupA.getMembers().add(unitC);
unitA.getGroups().add(unitGroupA);
unitB.getGroups().add(unitGroupA);
unitC.getGroups().add(unitGroupA);
unitGroupB.getMembers().add(unitA);
unitGroupB.getMembers().add(unitB);
unitGroupB.getMembers().add(unitF);
unitA.getGroups().add(unitGroupB);
unitB.getGroups().add(unitGroupB);
unitF.getGroups().add(unitGroupB);
unitGroupC.getMembers().add(unitA);
unitA.getGroups().add(unitGroupC);
organisationUnitGroupService.addOrganisationUnitGroup(unitGroupA);
organisationUnitGroupService.addOrganisationUnitGroup(unitGroupB);
organisationUnitGroupService.addOrganisationUnitGroup(unitGroupC);
organisationUnitGroupService.addOrganisationUnitGroup(unitGroupD);
// ---------------------------------------------------------------------
// Group sets
// ---------------------------------------------------------------------
unitGroupSetA = createOrganisationUnitGroupSet('A');
unitGroupSetB = createOrganisationUnitGroupSet('B');
unitGroupSetA.setCompulsory(true);
unitGroupSetB.setCompulsory(false);
unitGroupSetA.getOrganisationUnitGroups().add(unitGroupA);
unitGroupA.setGroupSet(unitGroupSetA);
unitGroupSetB.getOrganisationUnitGroups().add(unitGroupB);
unitGroupSetB.getOrganisationUnitGroups().add(unitGroupC);
unitGroupB.setGroupSet(unitGroupSetB);
unitGroupC.setGroupSet(unitGroupSetB);
organisationUnitGroupService.addOrganisationUnitGroupSet(unitGroupSetA);
organisationUnitGroupService.addOrganisationUnitGroupSet(unitGroupSetB);
}
@Override
public void setUpTest() {
aggregationService = (AggregationService) getBean(AggregationService.ID);
categoryService = (DataElementCategoryService) getBean(DataElementCategoryService.ID);
dataElementService = (DataElementService) getBean(DataElementService.ID);
indicatorService = (IndicatorService) getBean(IndicatorService.ID);
dataSetService = (DataSetService) getBean(DataSetService.ID);
periodService = (PeriodService) getBean(PeriodService.ID);
organisationUnitService = (OrganisationUnitService) getBean(OrganisationUnitService.ID);
dataValueService = (DataValueService) getBean(DataValueService.ID);
expressionService = (ExpressionService) getBean(ExpressionService.ID);
// ---------------------------------------------------------------------
// Setup Dimensions
// ---------------------------------------------------------------------
categoryOptionA = new DataElementCategoryOption("Male");
categoryOptionB = new DataElementCategoryOption("Female");
categoryService.addDataElementCategoryOption(categoryOptionA);
categoryService.addDataElementCategoryOption(categoryOptionB);
category = new DataElementCategory("Gender");
category.getCategoryOptions().add(categoryOptionA);
category.getCategoryOptions().add(categoryOptionB);
categoryService.addDataElementCategory(category);
categoryCombo = new DataElementCategoryCombo("Gender");
categoryCombo.getCategories().add(category);
categoryService.addDataElementCategoryCombo(categoryCombo);
categoryOptionComboA = createCategoryOptionCombo(categoryCombo, categoryOptionA);
categoryOptionComboB = createCategoryOptionCombo(categoryCombo, categoryOptionB);
categoryService.addDataElementCategoryOptionCombo(categoryOptionComboA);
categoryService.addDataElementCategoryOptionCombo(categoryOptionComboB);
// ---------------------------------------------------------------------
// Setup DataElements
// ---------------------------------------------------------------------
dataElementA =
createDataElement(
'A', DataElement.VALUE_TYPE_INT, DataElement.AGGREGATION_OPERATOR_SUM, categoryCombo);
dataElementB =
createDataElement(
'B', DataElement.VALUE_TYPE_BOOL, DataElement.AGGREGATION_OPERATOR_SUM, categoryCombo);
dataElementC =
createDataElement(
'C', DataElement.VALUE_TYPE_INT, DataElement.AGGREGATION_OPERATOR_SUM, categoryCombo);
dataElementService.addDataElement(dataElementA);
dataElementService.addDataElement(dataElementB);
dataElementService.addDataElement(dataElementC);
// ---------------------------------------------------------------------
// Setup Indicators
// ---------------------------------------------------------------------
indicatorType = createIndicatorType('A');
indicatorType.setFactor(100);
indicatorService.addIndicatorType(indicatorType);
indicatorA = createIndicator('A', indicatorType);
indicatorA.setNumerator(
"#{" + dataElementA.getUid() + SEPARATOR + categoryOptionComboA.getUid() + "}+150");
indicatorA.setDenominator(
"#{" + dataElementB.getUid() + SEPARATOR + categoryOptionComboA.getUid() + "}");
indicatorB = createIndicator('B', indicatorType);
indicatorB.setNumerator("#{" + dataElementC.getUid() + "}");
indicatorB.setDenominator("1");
indicatorService.addIndicator(indicatorA);
indicatorService.addIndicator(indicatorB);
// ---------------------------------------------------------------------
// Setup DataSets (to get correct PeriodType for DataElements)
// ---------------------------------------------------------------------
dataSet = createDataSet('A', new MonthlyPeriodType());
dataSet.getDataElements().add(dataElementA);
dataSet.getDataElements().add(dataElementB);
dataSetService.addDataSet(dataSet);
dataElementA.getDataSets().add(dataSet);
dataElementB.getDataSets().add(dataSet);
dataElementC.getDataSets().add(dataSet);
dataElementService.updateDataElement(dataElementA);
dataElementService.updateDataElement(dataElementB);
dataElementService.updateDataElement(dataElementC);
// ---------------------------------------------------------------------
// Setup Periods
// ---------------------------------------------------------------------
PeriodType monthly = new MonthlyPeriodType();
periodA = createPeriod(monthly, mar01, mar31);
periodB = createPeriod(monthly, apr01, apr30);
periodC = createPeriod(monthly, may01, may31);
periodService.addPeriod(periodA);
periodService.addPeriod(periodB);
periodService.addPeriod(periodC);
// ---------------------------------------------------------------------
// Setup OrganisationUnits
// ---------------------------------------------------------------------
unitA = createOrganisationUnit('A');
unitB = createOrganisationUnit('B', unitA);
unitC = createOrganisationUnit('C', unitA);
unitD = createOrganisationUnit('D', unitB);
unitE = createOrganisationUnit('E', unitB);
unitF = createOrganisationUnit('F', unitB);
unitG = createOrganisationUnit('G', unitF);
unitH = createOrganisationUnit('H', unitF);
unitI = createOrganisationUnit('I');
organisationUnitService.addOrganisationUnit(unitA);
organisationUnitService.addOrganisationUnit(unitB);
organisationUnitService.addOrganisationUnit(unitC);
organisationUnitService.addOrganisationUnit(unitD);
organisationUnitService.addOrganisationUnit(unitE);
organisationUnitService.addOrganisationUnit(unitF);
organisationUnitService.addOrganisationUnit(unitG);
organisationUnitService.addOrganisationUnit(unitH);
organisationUnitService.addOrganisationUnit(unitI);
// ---------------------------------------------------------------------
// Setup DataValues
// ---------------------------------------------------------------------
dataValueService.addDataValue(
createDataValue(dataElementA, periodA, unitC, "90", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodA, unitD, "10", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodA, unitE, "35", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodA, unitF, "25", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodA, unitG, "20", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodA, unitH, "60", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodB, unitC, "70", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodB, unitD, "40", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodB, unitE, "65", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodB, unitF, "55", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodB, unitG, "20", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodB, unitH, "15", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodC, unitC, "95", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodC, unitD, "40", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodC, unitE, "45", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodC, unitF, "30", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodC, unitG, "50", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementA, periodC, unitH, "70", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodA, unitC, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodA, unitD, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodA, unitE, F, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodA, unitF, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodA, unitG, F, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodA, unitH, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodB, unitC, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodB, unitD, F, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodB, unitE, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodB, unitF, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodB, unitG, F, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodB, unitH, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodC, unitC, F, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodC, unitD, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodC, unitE, F, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodC, unitF, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodC, unitG, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementB, periodC, unitH, T, categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementC, periodA, unitB, "30", categoryOptionComboA));
dataValueService.addDataValue(
createDataValue(dataElementC, periodA, unitB, "20", categoryOptionComboB));
aggregationService.clearCache();
}
@Transactional
public void export(
Collection<Integer> dataElementIds,
Collection<Integer> indicatorIds,
Collection<Integer> periodIds,
Collection<Integer> organisationUnitIds,
Collection<Integer> organisationUnitGroupIds,
TaskId id) {
final int cpuCores = SystemUtils.getCpuCores();
Clock clock =
new Clock()
.startClock()
.logTime(
"Data mart export process started, number of CPU cores: "
+ cpuCores
+ ", "
+ SystemUtils.getMemoryString());
notifier.clear(id, DATAMART).notify(id, DATAMART, "Data mart export process started");
// ---------------------------------------------------------------------
// Recreate temporary tables
// ---------------------------------------------------------------------
dataMartManager.dropTempAggregatedTables();
dataMartManager.createTempAggregatedTables();
clock.logTime("Recreated temporary tables");
// ---------------------------------------------------------------------
// Replace null with empty collection
// ---------------------------------------------------------------------
dataElementIds = dataElementIds != null ? dataElementIds : new ArrayList<Integer>();
indicatorIds = indicatorIds != null ? indicatorIds : new ArrayList<Integer>();
periodIds = periodIds != null ? periodIds : new ArrayList<Integer>();
organisationUnitIds =
organisationUnitIds != null ? organisationUnitIds : new ArrayList<Integer>();
organisationUnitGroupIds =
organisationUnitGroupIds != null ? organisationUnitGroupIds : new ArrayList<Integer>();
clock.logTime(
"Data elements: "
+ dataElementIds.size()
+ ", indicators: "
+ indicatorIds.size()
+ ", periods: "
+ periodIds.size()
+ ", org units: "
+ organisationUnitIds.size());
// ---------------------------------------------------------------------
// Get objects
// ---------------------------------------------------------------------
final Collection<Indicator> indicators = indicatorService.getIndicators(indicatorIds);
final Collection<Period> periods = periodService.getPeriods(periodIds);
final List<OrganisationUnit> organisationUnits =
new ArrayList<OrganisationUnit>(
organisationUnitService.getOrganisationUnits(organisationUnitIds));
final Collection<OrganisationUnitGroup> organisationUnitGroups =
organisationUnitGroupService.getOrganisationUnitGroups(organisationUnitGroupIds);
final Collection<DataElement> dataElements = dataElementService.getDataElements(dataElementIds);
clock.logTime("Retrieved meta-data objects");
notifier.notify(id, DATAMART, "Filtering meta-data");
// ---------------------------------------------------------------------
// Filter objects
// ---------------------------------------------------------------------
organisationUnitService.filterOrganisationUnitsWithoutData(organisationUnits);
FilterUtils.filter(dataElements, new AggregatableDataElementFilter());
FilterUtils.filter(dataElements, new DataElementWithAggregationFilter());
expressionService.filterInvalidIndicators(indicators);
clock.logTime("Filtered objects");
notifier.notify(id, DATAMART, "Loading indicators");
// ---------------------------------------------------------------------
// Explode indicator expressions
// ---------------------------------------------------------------------
for (Indicator indicator : indicators) {
indicator.setExplodedNumerator(expressionService.explodeExpression(indicator.getNumerator()));
indicator.setExplodedDenominator(
expressionService.explodeExpression(indicator.getDenominator()));
}
clock.logTime("Exploded indicator expressions");
notifier.notify(id, DATAMART, "Loading data elements");
// ---------------------------------------------------------------------
// Get operands
// ---------------------------------------------------------------------
final Collection<DataElementOperand> dataElementOperands =
categoryService.getOperands(dataElements);
final List<DataElementOperand> indicatorOperands =
new ArrayList<DataElementOperand>(
categoryService.populateOperands(
expressionService.getOperandsInIndicators(indicators)));
Set<DataElementOperand> allOperands = new HashSet<DataElementOperand>();
allOperands.addAll(dataElementOperands);
allOperands.addAll(indicatorOperands);
clock.logTime("Retrieved operands: " + allOperands.size());
notifier.notify(id, DATAMART, "Loading periods");
// ---------------------------------------------------------------------
// Filter out future periods
// ---------------------------------------------------------------------
FilterUtils.filter(periods, new PastAndCurrentPeriodFilter());
clock.logTime("Number of periods: " + periods.size());
notifier.notify(id, DATAMART, "Filtering data elements without data");
// ---------------------------------------------------------------------
// Remove operands without data
// ---------------------------------------------------------------------
allOperands = dataMartManager.getOperandsWithData(allOperands);
indicatorOperands.retainAll(allOperands);
clock.logTime(
"Number of operands with data: "
+ allOperands.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Populating crosstabulation table");
// ---------------------------------------------------------------------
// Create crosstabtable
// ---------------------------------------------------------------------
final Collection<Integer> intersectingPeriodIds =
ConversionUtils.getIdentifiers(Period.class, periodService.getIntersectionPeriods(periods));
final PeriodHierarchy periodHierarchy = periodService.getPeriodHierarchy(periods);
final Set<Integer> orgUnitChildrenIds =
organisationUnitService.getOrganisationUnitHierarchy().getChildren(organisationUnitIds);
final List<Integer> crossTabOrgUnitIds = new ArrayList<Integer>(orgUnitChildrenIds);
final String key = crossTabService.createCrossTabTable(crossTabOrgUnitIds);
final List<DataElementOperand> operandList = new ArrayList<DataElementOperand>(allOperands);
Collections.shuffle(operandList);
final List<List<DataElementOperand>> operandPages =
new PaginatedList<DataElementOperand>(operandList).setNumberOfPages(cpuCores).getPages();
List<Future<?>> crossTabFutures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
crossTabFutures.add(
crossTabService.populateCrossTabTable(
operandPage, intersectingPeriodIds, crossTabOrgUnitIds, key));
}
ConcurrentUtils.waitForCompletion(crossTabFutures);
clock.logTime("Populated crosstab table, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Exporting data element data");
final boolean isDataElements = true;
final boolean isIndicators = indicators != null && indicators.size() > 0;
final int groupLevel =
(Integer)
systemSettingManager.getSystemSetting(
KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL);
final boolean isGroups =
organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0;
if (isDataElements) {
// -----------------------------------------------------------------
// 1. Export data element values
// -----------------------------------------------------------------
if (operandList.size() > 0) {
final OrganisationUnitHierarchy orgUnitHierarchy =
organisationUnitService
.getOrganisationUnitHierarchy()
.prepareChildren(organisationUnits);
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
futures.add(
dataElementDataMart.exportDataValues(
operandPage,
periods,
organisationUnits,
null,
periodHierarchy,
orgUnitHierarchy,
AggregatedDataValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
}
clock.logTime(
"Exported values for data element operands ("
+ operandList.size()
+ "), pages: "
+ operandPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping data element index");
// -----------------------------------------------------------------
// 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropDataValueIndex();
clock.logTime("Dropped data element index");
notifier.notify(id, DATAMART, "Deleting existing data element data");
// -----------------------------------------------------------------
// 3. Delete existing aggregated data values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedDataValues(periodIds);
clock.logTime("Deleted existing data element data");
notifier.notify(id, DATAMART, "Copying data element data from temporary table");
// -----------------------------------------------------------------
// 4. Copy aggregated data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedDataValuesFromTemp();
clock.logTime("Copied data element data from temporary table");
notifier.notify(id, DATAMART, "Creating data element index");
// -----------------------------------------------------------------
// 5. Create data element index
// -----------------------------------------------------------------
dataMartManager.createDataValueIndex();
clock.logTime("Created data element index");
}
if (isGroups && isDataElements) {
// -----------------------------------------------------------------
// 1. Export data element values
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Exporting data element org unit data");
Collection<OrganisationUnit> groupOrganisationUnits =
new HashSet<OrganisationUnit>(organisationUnits);
FilterUtils.filter(
groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter(groupLevel));
if (operandList.size() > 0) {
final OrganisationUnitHierarchy orgUnitHierarchy =
organisationUnitService
.getOrganisationUnitHierarchy()
.prepareChildren(groupOrganisationUnits, organisationUnitGroups);
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
futures.add(
dataElementDataMart.exportDataValues(
operandPage,
periods,
groupOrganisationUnits,
organisationUnitGroups,
periodHierarchy,
orgUnitHierarchy,
AggregatedOrgUnitDataValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
}
clock.logTime(
"Exported values for data element operands ("
+ operandList.size()
+ "), pages: "
+ operandPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping data element data indexes");
// -----------------------------------------------------------------
// 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropOrgUnitDataValueIndex();
clock.logTime("Dropped org unit data element index");
notifier.notify(id, DATAMART, "Deleting existing org unit data element data");
// -----------------------------------------------------------------
// 3. Delete existing aggregated data values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitDataValues(periodIds);
clock.logTime("Deleted existing aggregated org unit datavalues");
notifier.notify(id, DATAMART, "Copying org unit data element data");
// -----------------------------------------------------------------
// 4. Copy aggregated org unit data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp();
clock.logTime("Copied org unit data element data from temporary table");
notifier.notify(id, DATAMART, "Creating org unit data element index");
// -----------------------------------------------------------------
// 5. Create org unit data element index
// -----------------------------------------------------------------
dataMartManager.createOrgUnitDataValueIndex();
clock.logTime("Created org unit data element index");
notifier.notify(id, DATAMART, "Exporting data for org unit indicator data");
}
crossTabService.dropCrossTabTable(key);
List<List<OrganisationUnit>> organisationUnitPages =
new PaginatedList<OrganisationUnit>(organisationUnits)
.setNumberOfPages(cpuCores)
.getPages();
if (isIndicators) {
// -----------------------------------------------------------------
// 1. Create and populate aggregated data cache
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Populating aggregated data cache");
crossTabService.createAggregatedDataCache(indicatorOperands, key);
List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
aggregatedDataCacheFutures.add(
crossTabService.populateAggregatedDataCache(
indicatorOperands, periods, organisationUnitPage, key));
}
ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);
clock.logTime(
"Created aggregated data cache, number of indicator operands: "
+ indicatorOperands.size());
notifier.notify(id, DATAMART, "Exporting indicator data");
// -----------------------------------------------------------------
// 2. Export indicator values
// -----------------------------------------------------------------
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
futures.add(
indicatorDataMart.exportIndicatorValues(
indicators,
periods,
organisationUnitPage,
null,
indicatorOperands,
AggregatedIndicatorValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
clock.logTime(
"Exported values for indicators ("
+ indicators.size()
+ "), pages: "
+ organisationUnitPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping indicator index");
// -----------------------------------------------------------------
// 3. Drop aggregated data cache and indicator index
// -----------------------------------------------------------------
crossTabService.dropAggregatedDataCache(key);
dataMartManager.dropIndicatorValueIndex();
clock.logTime("Dropped indicator index, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Deleting existing indicator data");
// -----------------------------------------------------------------
// 4. Delete existing aggregated indicator values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedIndicatorValues(periodIds);
clock.logTime("Deleted existing indicator data");
notifier.notify(id, DATAMART, "Copying indicator data from temporary table");
// -----------------------------------------------------------------
// 5. Copy aggregated data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedIndicatorValuesFromTemp();
clock.logTime("Copied indicator data from temporary table");
notifier.notify(id, DATAMART, "Creating indicator index");
// -----------------------------------------------------------------
// 6. Create indicator index
// -----------------------------------------------------------------
dataMartManager.createIndicatorValueIndex();
clock.logTime("Created indicator index");
}
if (isGroups && isIndicators) {
// -----------------------------------------------------------------
// 1. Create aggregated data cache
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Populating aggregated data cache");
crossTabService.createAggregatedOrgUnitDataCache(indicatorOperands, key);
List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
aggregatedDataCacheFutures.add(
crossTabService.populateAggregatedOrgUnitDataCache(
indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key));
}
ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);
clock.logTime("Created aggregated org unit data cache");
notifier.notify(id, DATAMART, "Exporting org unit indicator data");
// -----------------------------------------------------------------
// 2. Export indicator values
// -----------------------------------------------------------------
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
futures.add(
indicatorDataMart.exportIndicatorValues(
indicators,
periods,
organisationUnitPage,
organisationUnitGroups,
indicatorOperands,
AggregatedOrgUnitIndicatorValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
clock.logTime(
"Exported values for indicators ("
+ indicators.size()
+ "), pages: "
+ organisationUnitPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping org unit indicator index");
// -----------------------------------------------------------------
// 3. Drop aggregated data cache and indicator index
// -----------------------------------------------------------------
crossTabService.dropAggregatedOrgUnitDataCache(key);
dataMartManager.dropOrgUnitIndicatorValueIndex();
clock.logTime("Dropped org unit indicator index, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Deleting existing org unit indicator data");
// -----------------------------------------------------------------
// 4. Delete existing aggregated indicator values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitIndicatorValues(periodIds);
clock.logTime("Deleted existing aggregated org unit indicatorvalues");
notifier.notify(id, DATAMART, "Copying org unit indicator data from temporary table");
// -----------------------------------------------------------------
// 5. Copy aggregated org unit indicator values from temp table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();
clock.logTime("Copied org unit indicator data from temporary table");
notifier.notify(id, DATAMART, "Creating org unit indicator indexes");
// -----------------------------------------------------------------
// 6. Create org unit indicator index
// -----------------------------------------------------------------
dataMartManager.createOrgUnitIndicatorValueIndex();
clock.logTime("Created org unit indicator index");
}
// ---------------------------------------------------------------------
// Drop temporary tables
// ---------------------------------------------------------------------
dataMartManager.dropTempAggregatedTables();
clock.logTime("Dropped crosstab table");
clock.logTime("Data mart export process completed");
notifier.notify(id, DATAMART, INFO, "Data mart process completed", true);
}
/**
* Creates a table with the given indicator
*
* @param indicator The indicator
* @param i18n i18n object
* @param expressionService The expression service
* @param HEADER3 The header3 font
* @param ITALIC The italic font
* @param TEXT The text font
* @param keepTogether Indicates whether the table could be broken across multiple pages or should
* be kept at one page.
* @param columnWidths The column widths.
*/
public static PdfPTable printIndicator(
Indicator indicator,
I18n i18n,
ExpressionService expressionService,
boolean keepTogether,
float... columnWidths) {
PdfPTable table = getPdfPTable(keepTogether, columnWidths);
table.addCell(getHeaderCell(indicator.getName(), 2));
table.addCell(getEmptyCell(2, 15));
table.addCell(getItalicCell(i18n.getString("short_name")));
table.addCell(getTextCell(indicator.getShortName()));
if (nullIfEmpty(indicator.getCode()) != null) {
table.addCell(getItalicCell(i18n.getString("code")));
table.addCell(getTextCell(indicator.getCode()));
}
if (nullIfEmpty(indicator.getDescription()) != null) {
table.addCell(getItalicCell(i18n.getString("description")));
table.addCell(getTextCell(indicator.getDescription()));
}
table.addCell(getItalicCell(i18n.getString("annualized")));
table.addCell(getTextCell(i18n.getString(getBoolean().get(indicator.isAnnualized()))));
table.addCell(getItalicCell(i18n.getString("indicator_type")));
table.addCell(getTextCell(indicator.getIndicatorType().getName()));
table.addCell(getItalicCell(i18n.getString("numerator_description")));
table.addCell(getTextCell(indicator.getNumeratorDescription()));
table.addCell(getItalicCell(i18n.getString("numerator_formula")));
table.addCell(
getTextCell(expressionService.getExpressionDescription(indicator.getNumerator())));
table.addCell(getItalicCell(i18n.getString("denominator_description")));
table.addCell(getTextCell(indicator.getDenominatorDescription()));
table.addCell(getItalicCell(i18n.getString("denominator_formula")));
table.addCell(
getTextCell(expressionService.getExpressionDescription(indicator.getDenominator())));
for (AttributeValue value : indicator.getAttributeValues()) {
table.addCell(getItalicCell(value.getAttribute().getName()));
table.addCell(getTextCell(value.getValue()));
}
table.addCell(getEmptyCell(2, 30));
return table;
}
@Transactional
public void matchObject(int importObjectId, int existingObjectId) {
ImportObject importObject = importObjectStore.getImportObject(importObjectId);
Object object = importObject.getObject();
// ---------------------------------------------------------------------
// Updates the name of the import object to the name of the existing
// object.
// ---------------------------------------------------------------------
if (object.getClass().equals(DataElement.class)) {
DataElement element = (DataElement) object;
element.setName(dataElementService.getDataElement(existingObjectId).getName());
} else if (object.getClass().equals(DataElementGroup.class)) {
DataElementGroup group = (DataElementGroup) object;
group.setName(dataElementService.getDataElementGroup(existingObjectId).getName());
} else if (object.getClass().equals(DataElementGroupSet.class)) {
DataElementGroupSet groupSet = (DataElementGroupSet) object;
groupSet.setName(dataElementService.getDataElementGroupSet(existingObjectId).getName());
} else if (object.getClass().equals(IndicatorType.class)) {
IndicatorType type = (IndicatorType) object;
type.setName(indicatorService.getIndicatorType(existingObjectId).getName());
} else if (object.getClass().equals(Indicator.class)) {
Indicator indicator = (Indicator) object;
indicator.setName(indicatorService.getIndicator(existingObjectId).getName());
} else if (object.getClass().equals(IndicatorGroup.class)) {
IndicatorGroup group = (IndicatorGroup) object;
group.setName(indicatorService.getIndicatorGroup(existingObjectId).getName());
} else if (object.getClass().equals(IndicatorGroupSet.class)) {
IndicatorGroupSet groupSet = (IndicatorGroupSet) object;
groupSet.setName(indicatorService.getIndicatorGroupSet(existingObjectId).getName());
} else if (object.getClass().equals(DataDictionary.class)) {
DataDictionary dictionary = (DataDictionary) object;
dictionary.setName(dataDictionaryService.getDataDictionary(existingObjectId).getName());
} else if (object.getClass().equals(DataSet.class)) {
DataSet dataSet = (DataSet) object;
dataSet.setName(dataSetService.getDataSet(existingObjectId).getName());
} else if (object.getClass().equals(OrganisationUnit.class)) {
OrganisationUnit unit = (OrganisationUnit) object;
unit.setName(organisationUnitService.getOrganisationUnit(existingObjectId).getName());
} else if (object.getClass().equals(OrganisationUnitGroup.class)) {
OrganisationUnitGroup group = (OrganisationUnitGroup) object;
group.setName(
organisationUnitGroupService.getOrganisationUnitGroup(existingObjectId).getName());
} else if (object.getClass().equals(OrganisationUnitGroupSet.class)) {
OrganisationUnitGroupSet groupSet = (OrganisationUnitGroupSet) object;
groupSet.setName(
organisationUnitGroupService.getOrganisationUnitGroupSet(existingObjectId).getName());
} else if (object.getClass().equals(OrganisationUnitLevel.class)) {
OrganisationUnitLevel level = (OrganisationUnitLevel) object;
level.setName(organisationUnitService.getOrganisationUnitLevel(existingObjectId).getName());
} else if (object.getClass().equals(ValidationRule.class)) {
ValidationRule validationRule = (ValidationRule) object;
validationRule.setName(validationRuleService.getValidationRule(existingObjectId).getName());
} else if (object.getClass().equals(Report.class)) {
Report report = (Report) object;
report.setName(reportService.getReport(existingObjectId).getName());
} else if (object.getClass().equals(ReportTable.class)) {
ReportTable reportTable = (ReportTable) object;
reportTable.setName(reportTableService.getReportTable(existingObjectId).getName());
} else if (object.getClass().equals(Chart.class)) {
Chart chart = (Chart) object;
chart.setName(chartService.getChart(existingObjectId).getName());
} else if (object.getClass().equals(DataValue.class)) {
DataValue dataValue = (DataValue) object;
dataValue =
updateDataValue(
dataValue,
dataValueService.getDataValue(
dataValue.getSource(),
dataValue.getDataElement(),
dataValue.getPeriod(),
dataValue.getOptionCombo()));
}
// ---------------------------------------------------------------------
// Sets the status of the import object to match, these objects will
// later be ignored on import all but is needed for matching of
// associations.
// ---------------------------------------------------------------------
importObject.setStatus(ImportObjectStatus.MATCH);
importObjectStore.updateImportObject(importObject);
}
@Transactional
public void cascadeDeleteImportObject(int importObjectId) {
ImportObject importObject = importObjectStore.getImportObject(importObjectId);
if (importObject != null) {
if (importObject.getClassName().equals(DataElement.class.getName())) {
DataElement element = (DataElement) importObject.getObject();
deleteMemberAssociations(GroupMemberType.DATAELEMENTGROUP, element.getId());
deleteMemberAssociations(GroupMemberType.DATASET, element.getId());
deleteMemberAssociations(GroupMemberType.DATADICTIONARY_DATAELEMENT, element.getId());
deleteIndicatorsContainingDataElement(element.getId());
importDataValueService.deleteImportDataValuesByDataElement(element.getId());
} else if (importObject.getClassName().equals(DataElementGroup.class.getName())) {
DataElementGroup group = (DataElementGroup) importObject.getObject();
deleteGroupAssociations(GroupMemberType.DATAELEMENTGROUP, group.getId());
deleteMemberAssociations(GroupMemberType.DATAELEMENTGROUPSET, group.getId());
} else if (importObject.getClassName().equals(DataElementGroupSet.class.getName())) {
DataElementGroupSet groupSet = (DataElementGroupSet) importObject.getObject();
deleteGroupAssociations(GroupMemberType.DATAELEMENTGROUPSET, groupSet.getId());
} else if (importObject.getClassName().equals(IndicatorType.class.getName())) {
IndicatorType type = (IndicatorType) importObject.getObject();
deleteIndicatorsWithIndicatorType(type.getId());
} else if (importObject.getClassName().equals(Indicator.class.getName())) {
Indicator indicator = (Indicator) importObject.getObject();
deleteMemberAssociations(GroupMemberType.INDICATORGROUP, indicator.getId());
deleteMemberAssociations(GroupMemberType.DATADICTIONARY_INDICATOR, indicator.getId());
} else if (importObject.getClassName().equals(IndicatorGroup.class.getName())) {
IndicatorGroup group = (IndicatorGroup) importObject.getObject();
deleteGroupAssociations(GroupMemberType.INDICATORGROUP, group.getId());
deleteMemberAssociations(GroupMemberType.INDICATORGROUPSET, group.getId());
} else if (importObject.getClassName().equals(IndicatorGroupSet.class.getName())) {
IndicatorGroupSet groupSet = (IndicatorGroupSet) importObject.getObject();
deleteGroupAssociations(GroupMemberType.INDICATORGROUPSET, groupSet.getId());
} else if (importObject.getClassName().equals(DataDictionary.class.getName())) {
DataDictionary dictionary = (DataDictionary) importObject.getObject();
deleteGroupAssociations(GroupMemberType.DATADICTIONARY_DATAELEMENT, dictionary.getId());
deleteGroupAssociations(GroupMemberType.DATADICTIONARY_INDICATOR, dictionary.getId());
} else if (importObject.getClassName().equals(DataSet.class.getName())) {
DataSet dataSet = (DataSet) importObject.getObject();
deleteGroupAssociations(GroupMemberType.DATASET, dataSet.getId());
deleteMemberAssociations(GroupMemberType.DATASET_SOURCE, dataSet.getId());
deleteCompleteDataSetRegistrationsByDataSet(dataSet.getId());
} else if (importObject.getClassName().equals(OrganisationUnit.class.getName())) {
OrganisationUnit unit = (OrganisationUnit) importObject.getObject();
deleteMemberAssociations(GroupMemberType.ORGANISATIONUNITGROUP, unit.getId());
deleteGroupAssociations(GroupMemberType.ORGANISATIONUNITRELATIONSHIP, unit.getId());
deleteMemberAssociations(GroupMemberType.DATASET_SOURCE, unit.getId());
deleteMemberAssociations(GroupMemberType.ORGANISATIONUNITRELATIONSHIP, unit.getId());
deleteCompleteDataSetRegistrationsBySource(unit.getId());
importDataValueService.deleteImportDataValuesBySource(unit.getId());
} else if (importObject.getClassName().equals(OrganisationUnitGroup.class.getName())) {
OrganisationUnitGroup group = (OrganisationUnitGroup) importObject.getObject();
deleteGroupAssociations(GroupMemberType.ORGANISATIONUNITGROUP, group.getId());
deleteMemberAssociations(GroupMemberType.ORGANISATIONUNITGROUPSET, group.getId());
} else if (importObject.getClassName().equals(OrganisationUnitGroupSet.class.getName())) {
OrganisationUnitGroupSet groupSet = (OrganisationUnitGroupSet) importObject.getObject();
deleteGroupAssociations(GroupMemberType.ORGANISATIONUNITGROUPSET, groupSet.getId());
}
}
deleteImportObject(importObjectId);
}