@Override
protected void importUnique(ValidationRule object) {
expressionService.addExpression(object.getLeftSide());
expressionService.addExpression(object.getRightSide());
validationRuleService.saveValidationRule(object);
}
/**
* Validates a collection of validation rules.
*
* @param period the period to validate for.
* @param source the source to validate for.
* @param validationRules the rules to validate.
* @param dataElementsInRules the data elements which are part of the rules expressions.
* @param constantMap the constants which are part of the rule expressions.
* @param currentSize the current number of validation violations.
* @returns a collection of rules that did not pass validation.
*/
private Collection<ValidationResult> validateInternal(
Period period,
OrganisationUnit unit,
Collection<ValidationRule> validationRules,
Set<DataElement> dataElementsInRules,
Map<String, Double> constantMap,
int currentSize) {
Map<DataElementOperand, Double> valueMap =
dataValueService.getDataValueMap(dataElementsInRules, period, unit);
final Collection<ValidationResult> validationViolations = new HashSet<ValidationResult>();
if (currentSize < MAX_VIOLATIONS) {
Double leftSide = null;
Double rightSide = null;
boolean violation = false;
for (final ValidationRule validationRule : validationRules) {
if (validationRule.getPeriodType() != null
&& validationRule.getPeriodType().equals(period.getPeriodType())) {
Operator operator = validationRule.getOperator();
leftSide =
expressionService.getExpressionValue(
validationRule.getLeftSide(), valueMap, constantMap, null);
if (leftSide != null || Operator.compulsory_pair.equals(operator)) {
rightSide =
expressionService.getExpressionValue(
validationRule.getRightSide(), valueMap, constantMap, null);
if (rightSide != null || Operator.compulsory_pair.equals(operator)) {
if (Operator.compulsory_pair.equals(operator)) {
violation =
(leftSide != null && rightSide == null)
|| (leftSide == null && rightSide != null);
} else if (leftSide != null && rightSide != null) {
violation = !expressionIsTrue(leftSide, operator, rightSide);
}
if (violation) {
validationViolations.add(
new ValidationResult(
period,
unit,
validationRule,
getRounded(zeroIfNull(leftSide), DECIMALS),
getRounded(zeroIfNull(rightSide), DECIMALS)));
}
}
}
}
}
}
return validationViolations;
}
/**
* Creates a table with the given indicator
*
* @param indicator The indicator
* @param i18n i18n object
* @param expressionService The expression service
* @param HEADER3 The header3 font
* @param ITALIC The italic font
* @param TEXT The text font
* @param keepTogether Indicates whether the table could be broken across multiple pages or should
* be kept at one page.
* @param columnWidths The column widths.
*/
public static PdfPTable printIndicator(
Indicator indicator,
I18n i18n,
ExpressionService expressionService,
boolean keepTogether,
float... columnWidths) {
PdfPTable table = getPdfPTable(keepTogether, columnWidths);
table.addCell(getHeaderCell(indicator.getName(), 2));
table.addCell(getEmptyCell(2, 15));
table.addCell(getItalicCell(i18n.getString("short_name")));
table.addCell(getTextCell(indicator.getShortName()));
if (nullIfEmpty(indicator.getCode()) != null) {
table.addCell(getItalicCell(i18n.getString("code")));
table.addCell(getTextCell(indicator.getCode()));
}
if (nullIfEmpty(indicator.getDescription()) != null) {
table.addCell(getItalicCell(i18n.getString("description")));
table.addCell(getTextCell(indicator.getDescription()));
}
table.addCell(getItalicCell(i18n.getString("annualized")));
table.addCell(getTextCell(i18n.getString(getBoolean().get(indicator.isAnnualized()))));
table.addCell(getItalicCell(i18n.getString("indicator_type")));
table.addCell(getTextCell(indicator.getIndicatorType().getName()));
table.addCell(getItalicCell(i18n.getString("numerator_description")));
table.addCell(getTextCell(indicator.getNumeratorDescription()));
table.addCell(getItalicCell(i18n.getString("numerator_formula")));
table.addCell(
getTextCell(expressionService.getExpressionDescription(indicator.getNumerator())));
table.addCell(getItalicCell(i18n.getString("denominator_description")));
table.addCell(getTextCell(indicator.getDenominatorDescription()));
table.addCell(getItalicCell(i18n.getString("denominator_formula")));
table.addCell(
getTextCell(expressionService.getExpressionDescription(indicator.getDenominator())));
for (AttributeValue value : indicator.getAttributeValues()) {
table.addCell(getItalicCell(value.getAttribute().getName()));
table.addCell(getTextCell(value.getValue()));
}
table.addCell(getEmptyCell(2, 30));
return table;
}
/**
* Creates a table with the given validation rule
*
* @param validationRule The validation rule
* @param i18n i18n object
* @param expressionService The expression service
* @param HEADER3 The header3 font
* @param ITALIC The italic font
* @param TEXT The text font
* @param keepTogether Indicates whether the table could be broken across multiple pages or should
* be kept at one page.
* @param columnWidths The column widths.
*/
public static PdfPTable printValidationRule(
ValidationRule validationRule,
I18n i18n,
ExpressionService expressionService,
boolean keepTogether,
float... columnWidths) {
PdfPTable table = getPdfPTable(keepTogether, columnWidths);
table.addCell(getHeaderCell(validationRule.getName(), 2));
table.addCell(getEmptyCell(2, 15));
if (nullIfEmpty(validationRule.getDescription()) != null) {
table.addCell(getItalicCell(i18n.getString("description")));
table.addCell(getTextCell(validationRule.getDescription()));
}
table.addCell(getItalicCell(i18n.getString("operator")));
table.addCell(getTextCell(i18n.getString(validationRule.getOperator().toString())));
table.addCell(getItalicCell(i18n.getString("left_side_of_expression")));
table.addCell(
getTextCell(
expressionService.getExpressionDescription(
validationRule.getLeftSide().getExpression())));
table.addCell(getItalicCell(i18n.getString("left_side_description")));
table.addCell(getTextCell(validationRule.getLeftSide().getDescription()));
table.addCell(getItalicCell(i18n.getString("right_side_of_expression")));
table.addCell(
getTextCell(
expressionService.getExpressionDescription(
validationRule.getRightSide().getExpression())));
table.addCell(getItalicCell(i18n.getString("right_side_description")));
table.addCell(getTextCell(validationRule.getRightSide().getDescription()));
table.addCell(getItalicCell(i18n.getString("period_type")));
table.addCell(getTextCell(i18n.getString(validationRule.getPeriodType().getName())));
table.addCell(getEmptyCell(2, 30));
return table;
}
public Collection<ValidationRule> getRelevantValidationRules(DataSet dataSet) {
Set<ValidationRule> relevantValidationRules = new HashSet<ValidationRule>();
Set<DataElementOperand> operands = dataEntryFormService.getOperandsInDataEntryForm(dataSet);
Set<DataElementOperand> validationRuleOperands = new HashSet<DataElementOperand>();
for (ValidationRule validationRule : getAllValidationRules()) {
validationRuleOperands.clear();
validationRuleOperands.addAll(
expressionService.getOperandsInExpression(validationRule.getLeftSide().getExpression()));
validationRuleOperands.addAll(
expressionService.getOperandsInExpression(validationRule.getRightSide().getExpression()));
if (operands.containsAll(validationRuleOperands)) {
relevantValidationRules.add(validationRule);
}
}
return relevantValidationRules;
}
@Override
protected void importMatching(ValidationRule object, ValidationRule match) {
match.setName(object.getName());
match.setDescription(object.getDescription());
match.setOperator(object.getOperator());
match.getLeftSide().setExpression(object.getLeftSide().getExpression());
match.getLeftSide().setDescription(object.getLeftSide().getDescription());
match
.getLeftSide()
.setDataElementsInExpression(object.getLeftSide().getDataElementsInExpression());
match.getRightSide().setExpression(object.getRightSide().getExpression());
match.getRightSide().setDescription(object.getRightSide().getDescription());
match
.getRightSide()
.setDataElementsInExpression(object.getRightSide().getDataElementsInExpression());
expressionService.updateExpression(match.getLeftSide());
expressionService.updateExpression(match.getRightSide());
validationRuleService.updateValidationRule(match);
}
public String execute() {
Expression leftSide = new Expression();
leftSide.setExpression(leftSideExpression);
leftSide.setDescription(leftSideDescription);
leftSide.setNullIfBlank(leftSideNullIfBlank);
leftSide.setDataElementsInExpression(
expressionService.getDataElementsInExpression(leftSideExpression));
leftSide.setOptionCombosInExpression(
expressionService.getOptionCombosInExpression(leftSideExpression));
Expression rightSide = new Expression();
rightSide.setExpression(rightSideExpression);
rightSide.setDescription(rightSideDescription);
rightSide.setNullIfBlank(rightSideNullIfBlank);
rightSide.setDataElementsInExpression(
expressionService.getDataElementsInExpression(rightSideExpression));
rightSide.setOptionCombosInExpression(
expressionService.getOptionCombosInExpression(rightSideExpression));
ValidationRule validationRule = new ValidationRule();
validationRule.setName(name);
validationRule.setDescription(description);
validationRule.setType(ValidationRule.TYPE_ABSOLUTE);
validationRule.setOperator(Operator.valueOf(operator));
validationRule.setLeftSide(leftSide);
validationRule.setRightSide(rightSide);
PeriodType periodType = periodService.getPeriodTypeByName(periodTypeName);
validationRule.setPeriodType(periodType);
validationRuleService.saveValidationRule(validationRule);
return SUCCESS;
}
protected void addDataValue(
OrganisationUnit unit,
Period period,
String expression,
String value,
Set<DataValue> oldList,
Set<DataValue> newList) {
// value = value.replaceAll( "\\.", "" ).replace( ",", "." );
DataElementOperand operand =
expressionService.getOperandsInExpression(expression).iterator().next();
DataElement dataElement = dataElementService.getDataElement(operand.getDataElementId());
DataElementCategoryOptionCombo optionCombo =
categoryService.getDataElementCategoryOptionCombo(operand.getOptionComboId());
String storedBy = currentUserService.getCurrentUsername();
DataValue dataValue = dataValueService.getDataValue(unit, dataElement, period, optionCombo);
if (dataValue == null) {
dataValue =
new DataValue(dataElement, period, unit, value, storedBy, new Date(), null, optionCombo);
dataValueService.addDataValue(dataValue);
newList.add(dataValue);
} else {
DataValue backedUpDataValue =
new DataValue(dataElement, period, unit, dataValue.getValue(), optionCombo);
oldList.add(backedUpDataValue);
dataValue.setValue(value);
dataValue.setTimestamp(new Date());
dataValue.setStoredBy(storedBy);
dataValueService.updateDataValue(dataValue);
}
}
@Transactional
public void export(
Collection<Integer> dataElementIds,
Collection<Integer> indicatorIds,
Collection<Integer> periodIds,
Collection<Integer> organisationUnitIds,
Collection<Integer> organisationUnitGroupIds,
TaskId id) {
final int cpuCores = SystemUtils.getCpuCores();
Clock clock =
new Clock()
.startClock()
.logTime(
"Data mart export process started, number of CPU cores: "
+ cpuCores
+ ", "
+ SystemUtils.getMemoryString());
notifier.clear(id, DATAMART).notify(id, DATAMART, "Data mart export process started");
// ---------------------------------------------------------------------
// Recreate temporary tables
// ---------------------------------------------------------------------
dataMartManager.dropTempAggregatedTables();
dataMartManager.createTempAggregatedTables();
clock.logTime("Recreated temporary tables");
// ---------------------------------------------------------------------
// Replace null with empty collection
// ---------------------------------------------------------------------
dataElementIds = dataElementIds != null ? dataElementIds : new ArrayList<Integer>();
indicatorIds = indicatorIds != null ? indicatorIds : new ArrayList<Integer>();
periodIds = periodIds != null ? periodIds : new ArrayList<Integer>();
organisationUnitIds =
organisationUnitIds != null ? organisationUnitIds : new ArrayList<Integer>();
organisationUnitGroupIds =
organisationUnitGroupIds != null ? organisationUnitGroupIds : new ArrayList<Integer>();
clock.logTime(
"Data elements: "
+ dataElementIds.size()
+ ", indicators: "
+ indicatorIds.size()
+ ", periods: "
+ periodIds.size()
+ ", org units: "
+ organisationUnitIds.size());
// ---------------------------------------------------------------------
// Get objects
// ---------------------------------------------------------------------
final Collection<Indicator> indicators = indicatorService.getIndicators(indicatorIds);
final Collection<Period> periods = periodService.getPeriods(periodIds);
final List<OrganisationUnit> organisationUnits =
new ArrayList<OrganisationUnit>(
organisationUnitService.getOrganisationUnits(organisationUnitIds));
final Collection<OrganisationUnitGroup> organisationUnitGroups =
organisationUnitGroupService.getOrganisationUnitGroups(organisationUnitGroupIds);
final Collection<DataElement> dataElements = dataElementService.getDataElements(dataElementIds);
clock.logTime("Retrieved meta-data objects");
notifier.notify(id, DATAMART, "Filtering meta-data");
// ---------------------------------------------------------------------
// Filter objects
// ---------------------------------------------------------------------
organisationUnitService.filterOrganisationUnitsWithoutData(organisationUnits);
FilterUtils.filter(dataElements, new AggregatableDataElementFilter());
FilterUtils.filter(dataElements, new DataElementWithAggregationFilter());
expressionService.filterInvalidIndicators(indicators);
clock.logTime("Filtered objects");
notifier.notify(id, DATAMART, "Loading indicators");
// ---------------------------------------------------------------------
// Explode indicator expressions
// ---------------------------------------------------------------------
for (Indicator indicator : indicators) {
indicator.setExplodedNumerator(expressionService.explodeExpression(indicator.getNumerator()));
indicator.setExplodedDenominator(
expressionService.explodeExpression(indicator.getDenominator()));
}
clock.logTime("Exploded indicator expressions");
notifier.notify(id, DATAMART, "Loading data elements");
// ---------------------------------------------------------------------
// Get operands
// ---------------------------------------------------------------------
final Collection<DataElementOperand> dataElementOperands =
categoryService.getOperands(dataElements);
final List<DataElementOperand> indicatorOperands =
new ArrayList<DataElementOperand>(
categoryService.populateOperands(
expressionService.getOperandsInIndicators(indicators)));
Set<DataElementOperand> allOperands = new HashSet<DataElementOperand>();
allOperands.addAll(dataElementOperands);
allOperands.addAll(indicatorOperands);
clock.logTime("Retrieved operands: " + allOperands.size());
notifier.notify(id, DATAMART, "Loading periods");
// ---------------------------------------------------------------------
// Filter out future periods
// ---------------------------------------------------------------------
FilterUtils.filter(periods, new PastAndCurrentPeriodFilter());
clock.logTime("Number of periods: " + periods.size());
notifier.notify(id, DATAMART, "Filtering data elements without data");
// ---------------------------------------------------------------------
// Remove operands without data
// ---------------------------------------------------------------------
allOperands = dataMartManager.getOperandsWithData(allOperands);
indicatorOperands.retainAll(allOperands);
clock.logTime(
"Number of operands with data: "
+ allOperands.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Populating crosstabulation table");
// ---------------------------------------------------------------------
// Create crosstabtable
// ---------------------------------------------------------------------
final Collection<Integer> intersectingPeriodIds =
ConversionUtils.getIdentifiers(Period.class, periodService.getIntersectionPeriods(periods));
final PeriodHierarchy periodHierarchy = periodService.getPeriodHierarchy(periods);
final Set<Integer> orgUnitChildrenIds =
organisationUnitService.getOrganisationUnitHierarchy().getChildren(organisationUnitIds);
final List<Integer> crossTabOrgUnitIds = new ArrayList<Integer>(orgUnitChildrenIds);
final String key = crossTabService.createCrossTabTable(crossTabOrgUnitIds);
final List<DataElementOperand> operandList = new ArrayList<DataElementOperand>(allOperands);
Collections.shuffle(operandList);
final List<List<DataElementOperand>> operandPages =
new PaginatedList<DataElementOperand>(operandList).setNumberOfPages(cpuCores).getPages();
List<Future<?>> crossTabFutures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
crossTabFutures.add(
crossTabService.populateCrossTabTable(
operandPage, intersectingPeriodIds, crossTabOrgUnitIds, key));
}
ConcurrentUtils.waitForCompletion(crossTabFutures);
clock.logTime("Populated crosstab table, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Exporting data element data");
final boolean isDataElements = true;
final boolean isIndicators = indicators != null && indicators.size() > 0;
final int groupLevel =
(Integer)
systemSettingManager.getSystemSetting(
KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL);
final boolean isGroups =
organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0;
if (isDataElements) {
// -----------------------------------------------------------------
// 1. Export data element values
// -----------------------------------------------------------------
if (operandList.size() > 0) {
final OrganisationUnitHierarchy orgUnitHierarchy =
organisationUnitService
.getOrganisationUnitHierarchy()
.prepareChildren(organisationUnits);
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
futures.add(
dataElementDataMart.exportDataValues(
operandPage,
periods,
organisationUnits,
null,
periodHierarchy,
orgUnitHierarchy,
AggregatedDataValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
}
clock.logTime(
"Exported values for data element operands ("
+ operandList.size()
+ "), pages: "
+ operandPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping data element index");
// -----------------------------------------------------------------
// 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropDataValueIndex();
clock.logTime("Dropped data element index");
notifier.notify(id, DATAMART, "Deleting existing data element data");
// -----------------------------------------------------------------
// 3. Delete existing aggregated data values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedDataValues(periodIds);
clock.logTime("Deleted existing data element data");
notifier.notify(id, DATAMART, "Copying data element data from temporary table");
// -----------------------------------------------------------------
// 4. Copy aggregated data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedDataValuesFromTemp();
clock.logTime("Copied data element data from temporary table");
notifier.notify(id, DATAMART, "Creating data element index");
// -----------------------------------------------------------------
// 5. Create data element index
// -----------------------------------------------------------------
dataMartManager.createDataValueIndex();
clock.logTime("Created data element index");
}
if (isGroups && isDataElements) {
// -----------------------------------------------------------------
// 1. Export data element values
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Exporting data element org unit data");
Collection<OrganisationUnit> groupOrganisationUnits =
new HashSet<OrganisationUnit>(organisationUnits);
FilterUtils.filter(
groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter(groupLevel));
if (operandList.size() > 0) {
final OrganisationUnitHierarchy orgUnitHierarchy =
organisationUnitService
.getOrganisationUnitHierarchy()
.prepareChildren(groupOrganisationUnits, organisationUnitGroups);
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
futures.add(
dataElementDataMart.exportDataValues(
operandPage,
periods,
groupOrganisationUnits,
organisationUnitGroups,
periodHierarchy,
orgUnitHierarchy,
AggregatedOrgUnitDataValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
}
clock.logTime(
"Exported values for data element operands ("
+ operandList.size()
+ "), pages: "
+ operandPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping data element data indexes");
// -----------------------------------------------------------------
// 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropOrgUnitDataValueIndex();
clock.logTime("Dropped org unit data element index");
notifier.notify(id, DATAMART, "Deleting existing org unit data element data");
// -----------------------------------------------------------------
// 3. Delete existing aggregated data values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitDataValues(periodIds);
clock.logTime("Deleted existing aggregated org unit datavalues");
notifier.notify(id, DATAMART, "Copying org unit data element data");
// -----------------------------------------------------------------
// 4. Copy aggregated org unit data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp();
clock.logTime("Copied org unit data element data from temporary table");
notifier.notify(id, DATAMART, "Creating org unit data element index");
// -----------------------------------------------------------------
// 5. Create org unit data element index
// -----------------------------------------------------------------
dataMartManager.createOrgUnitDataValueIndex();
clock.logTime("Created org unit data element index");
notifier.notify(id, DATAMART, "Exporting data for org unit indicator data");
}
crossTabService.dropCrossTabTable(key);
List<List<OrganisationUnit>> organisationUnitPages =
new PaginatedList<OrganisationUnit>(organisationUnits)
.setNumberOfPages(cpuCores)
.getPages();
if (isIndicators) {
// -----------------------------------------------------------------
// 1. Create and populate aggregated data cache
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Populating aggregated data cache");
crossTabService.createAggregatedDataCache(indicatorOperands, key);
List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
aggregatedDataCacheFutures.add(
crossTabService.populateAggregatedDataCache(
indicatorOperands, periods, organisationUnitPage, key));
}
ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);
clock.logTime(
"Created aggregated data cache, number of indicator operands: "
+ indicatorOperands.size());
notifier.notify(id, DATAMART, "Exporting indicator data");
// -----------------------------------------------------------------
// 2. Export indicator values
// -----------------------------------------------------------------
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
futures.add(
indicatorDataMart.exportIndicatorValues(
indicators,
periods,
organisationUnitPage,
null,
indicatorOperands,
AggregatedIndicatorValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
clock.logTime(
"Exported values for indicators ("
+ indicators.size()
+ "), pages: "
+ organisationUnitPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping indicator index");
// -----------------------------------------------------------------
// 3. Drop aggregated data cache and indicator index
// -----------------------------------------------------------------
crossTabService.dropAggregatedDataCache(key);
dataMartManager.dropIndicatorValueIndex();
clock.logTime("Dropped indicator index, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Deleting existing indicator data");
// -----------------------------------------------------------------
// 4. Delete existing aggregated indicator values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedIndicatorValues(periodIds);
clock.logTime("Deleted existing indicator data");
notifier.notify(id, DATAMART, "Copying indicator data from temporary table");
// -----------------------------------------------------------------
// 5. Copy aggregated data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedIndicatorValuesFromTemp();
clock.logTime("Copied indicator data from temporary table");
notifier.notify(id, DATAMART, "Creating indicator index");
// -----------------------------------------------------------------
// 6. Create indicator index
// -----------------------------------------------------------------
dataMartManager.createIndicatorValueIndex();
clock.logTime("Created indicator index");
}
if (isGroups && isIndicators) {
// -----------------------------------------------------------------
// 1. Create aggregated data cache
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Populating aggregated data cache");
crossTabService.createAggregatedOrgUnitDataCache(indicatorOperands, key);
List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
aggregatedDataCacheFutures.add(
crossTabService.populateAggregatedOrgUnitDataCache(
indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key));
}
ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);
clock.logTime("Created aggregated org unit data cache");
notifier.notify(id, DATAMART, "Exporting org unit indicator data");
// -----------------------------------------------------------------
// 2. Export indicator values
// -----------------------------------------------------------------
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
futures.add(
indicatorDataMart.exportIndicatorValues(
indicators,
periods,
organisationUnitPage,
organisationUnitGroups,
indicatorOperands,
AggregatedOrgUnitIndicatorValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
clock.logTime(
"Exported values for indicators ("
+ indicators.size()
+ "), pages: "
+ organisationUnitPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping org unit indicator index");
// -----------------------------------------------------------------
// 3. Drop aggregated data cache and indicator index
// -----------------------------------------------------------------
crossTabService.dropAggregatedOrgUnitDataCache(key);
dataMartManager.dropOrgUnitIndicatorValueIndex();
clock.logTime("Dropped org unit indicator index, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Deleting existing org unit indicator data");
// -----------------------------------------------------------------
// 4. Delete existing aggregated indicator values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitIndicatorValues(periodIds);
clock.logTime("Deleted existing aggregated org unit indicatorvalues");
notifier.notify(id, DATAMART, "Copying org unit indicator data from temporary table");
// -----------------------------------------------------------------
// 5. Copy aggregated org unit indicator values from temp table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();
clock.logTime("Copied org unit indicator data from temporary table");
notifier.notify(id, DATAMART, "Creating org unit indicator indexes");
// -----------------------------------------------------------------
// 6. Create org unit indicator index
// -----------------------------------------------------------------
dataMartManager.createOrgUnitIndicatorValueIndex();
clock.logTime("Created org unit indicator index");
}
// ---------------------------------------------------------------------
// Drop temporary tables
// ---------------------------------------------------------------------
dataMartManager.dropTempAggregatedTables();
clock.logTime("Dropped crosstab table");
clock.logTime("Data mart export process completed");
notifier.notify(id, DATAMART, INFO, "Data mart process completed", true);
}