@Override
public String execute() throws Exception {
CategoryOptionGroupSet categoryOptionGroupSet =
dataElementCategoryService.getCategoryOptionGroupSet(id);
dataElementCategoryService.deleteCategoryOptionGroupSet(categoryOptionGroupSet);
return SUCCESS;
}
@Override
public void setUpTest() {
categoryCombo =
categoryService.getDataElementCategoryComboByName(
DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME);
categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo();
dataElementIds = new HashSet<>();
indicatorIds = new HashSet<>();
periodIds = new HashSet<>();
organisationUnitIds = new HashSet<>();
}
@Override
public DataApprovalLevel getLowestDataApprovalLevel(
OrganisationUnit orgUnit, DataElementCategoryOptionCombo attributeOptionCombo) {
Set<CategoryOptionGroupSet> cogSets = null;
if (attributeOptionCombo != null
&& attributeOptionCombo != categoryService.getDefaultDataElementCategoryOptionCombo()) {
cogSets = new HashSet<>();
for (DataElementCategoryOption option : attributeOptionCombo.getCategoryOptions()) {
if (option.getGroupSets() != null) {
cogSets.addAll(option.getGroupSets());
}
}
}
int orgUnitLevel = orgUnit.getLevel();
List<DataApprovalLevel> approvalLevels = getDataApprovalLevelsByOrgUnitLevel(orgUnitLevel);
for (DataApprovalLevel level : Lists.reverse(approvalLevels)) {
if (level.getCategoryOptionGroupSet() == null) {
if (cogSets == null) {
return level;
}
} else if (cogSets != null && cogSets.contains(level.getCategoryOptionGroupSet())) {
return level;
}
}
return null;
}
public String execute() {
if (dataSetId != null) {
dataSet = dataSetService.getDataSet(dataSetId);
operands =
new ArrayList<DataElementOperand>(
dataElementCategoryService.getFullOperands(dataSet.getDataElements()));
} else {
operands = new ArrayList<DataElementOperand>();
}
if (key != null) {
Iterator<DataElementOperand> iterator = operands.iterator();
while (iterator.hasNext()) {
DataElementOperand operand = iterator.next();
if (operand.getOperandName().toLowerCase().indexOf(key.toLowerCase()) == -1) {
iterator.remove();
}
}
}
Collections.sort(operands, new DataElementOperandNameComparator());
if (usePaging) {
this.paging = createPaging(operands.size());
operands = operands.subList(paging.getStartPos(), paging.getEndPos());
}
return SUCCESS;
}
public String execute() {
try {
DataElementCategory category = dataElementCategoryService.getDataElementCategory(id);
dataElementCategoryService.deleteDataElementCategory(category);
} catch (DeleteNotAllowedException ex) {
if (ex.getErrorCode().equals(DeleteNotAllowedException.ERROR_ASSOCIATED_BY_OTHER_OBJECTS)) {
message =
i18n.getString("object_not_deleted_associated_by_objects") + " " + ex.getMessage();
return ERROR;
}
}
return SUCCESS;
}
@Override
protected void importMatching(DataElementCategoryOption object, DataElementCategoryOption match) {
match.setCode(object.getCode());
match.setUid(object.getUid());
match.setName(object.getName());
categoryService.updateDataElementCategoryOption(match);
}
@Override
public String execute() {
dataElementCategories =
new ArrayList<>(dataElementCategoryService.getAllDataElementCategories());
Collections.sort(dataElementCategories, IdentifiableObjectNameComparator.INSTANCE);
return SUCCESS;
}
/**
* Can the user read from this CategoryOptionGroupSet (COGS)?
*
* <p>If the COGS is null, then the user must have no dimension constraints. (In other words, the
* user must be able to read across all category option groups.)
*
* <p>If the COGS is not null, then the user must be able to read at least one category option
* group from the category option group set.
*
* @param cogs The category option group set to test
* @return true if user can read at least one category option group.
*/
private boolean canReadCOGS(User user, CategoryOptionGroupSet cogs) {
if (cogs == null) {
UserCredentials userCredentials = user.getUserCredentials();
return CollectionUtils.isEmpty(userCredentials.getCogsDimensionConstraints())
&& CollectionUtils.isEmpty(userCredentials.getCatDimensionConstraints());
}
return !CollectionUtils.isEmpty(categoryService.getCategoryOptionGroups(cogs));
}
public String execute() throws Exception {
// ---------------------------------------------------------------------
// Prepare values
// ---------------------------------------------------------------------
code = nullIfEmpty(code);
shortName = nullIfEmpty(shortName);
description = nullIfEmpty(description);
PeriodType periodType = PeriodType.getPeriodTypeByName(frequencySelect);
DataSet dataSet = new DataSet(name, shortName, code, periodType);
MapLegendSet legendSet = mappingService.getMapLegendSet(selectedLegendSetId);
dataSet.setExpiryDays(expiryDays);
dataSet.setTimelyDays(timelyDays);
dataSet.setSkipAggregation(skipAggregation);
for (String id : dataElementsSelectedList) {
dataSet.addDataElement(dataElementService.getDataElement(Integer.parseInt(id)));
}
Set<Indicator> indicators = new HashSet<Indicator>();
for (String id : indicatorsSelectedList) {
indicators.add(indicatorService.getIndicator(Integer.parseInt(id)));
}
if (categoryComboId != null) {
dataSet.setCategoryCombo(categoryService.getDataElementCategoryCombo(categoryComboId));
}
dataSet.setDescription(description);
dataSet.setVersion(1);
dataSet.setMobile(false);
dataSet.setIndicators(indicators);
dataSet.setNotificationRecipients(userGroupService.getUserGroup(notificationRecipients));
dataSet.setAllowFuturePeriods(allowFuturePeriods);
dataSet.setFieldCombinationRequired(fieldCombinationRequired);
dataSet.setValidCompleteOnly(validCompleteOnly);
dataSet.setNotifyCompletingUser(notifyCompletingUser);
dataSet.setApproveData(approveData);
dataSet.setSkipOffline(skipOffline);
dataSet.setDataElementDecoration(dataElementDecoration);
dataSet.setRenderAsTabs(renderAsTabs);
dataSet.setRenderHorizontally(renderHorizontally);
dataSet.setLegendSet(legendSet);
dataSetService.addDataSet(dataSet);
userService.assignDataSetToUserRole(dataSet);
return SUCCESS;
}
@Override
public String execute() {
DataElementCategoryCombo categoryCombo =
dataElementCategoryService.getDataElementCategoryCombo(id);
DataElementCategoryCombo defaultCategoryCombo =
dataElementCategoryService.getDataElementCategoryComboByName(
DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME);
if (!categoryCombo.equals(defaultCategoryCombo)) {
try {
dataElementCategoryService.deleteDataElementCategoryCombo(categoryCombo);
} catch (DeleteNotAllowedException ex) {
if (ex.getErrorCode().equals(DeleteNotAllowedException.ERROR_ASSOCIATED_BY_OTHER_OBJECTS)) {
message =
i18n.getString("object_not_deleted_associated_by_objects") + " " + ex.getMessage();
return ERROR;
}
}
}
return SUCCESS;
}
@Override
public Set<CategoryOptionGroup> getCogDimensionConstraints(UserCredentials userCredentials) {
Set<CategoryOptionGroup> groups = null;
Set<CategoryOptionGroupSet> cogsConstraints = userCredentials.getCogsDimensionConstraints();
if (cogsConstraints != null && !cogsConstraints.isEmpty()) {
groups = new HashSet<>();
for (CategoryOptionGroupSet cogs : cogsConstraints) {
groups.addAll(categoryService.getCategoryOptionGroups(cogs));
}
}
return groups;
}
@Override
public Set<DataElementCategoryOption> getCoDimensionConstraints(UserCredentials userCredentials) {
Set<DataElementCategoryOption> options = null;
Set<DataElementCategory> catConstraints = userCredentials.getCatDimensionConstraints();
if (catConstraints != null && !catConstraints.isEmpty()) {
options = new HashSet<>();
for (DataElementCategory category : catConstraints) {
options.addAll(categoryService.getDataElementCategoryOptions(category));
}
}
return options;
}
public String execute() {
if (name != null) {
DataElementCategoryOption match =
dataElementCategoryService.getDataElementCategoryOptionByName(name);
if (match != null && (id == null || match.getId() != id)) {
message = i18n.getString("name_in_use");
return ERROR;
}
}
message = "ok";
return SUCCESS;
}
@Override
public List<DataApprovalLevel> getUserDataApprovalLevels() {
UserCredentials userCredentials = currentUserService.getCurrentUser().getUserCredentials();
boolean mayApprove = userCredentials.isAuthorized(DataApproval.AUTH_APPROVE);
boolean mayApproveAtLowerLevels =
userCredentials.isAuthorized(DataApproval.AUTH_APPROVE_LOWER_LEVELS);
boolean mayAcceptAtLowerLevels =
userCredentials.isAuthorized(DataApproval.AUTH_ACCEPT_LOWER_LEVELS);
if (!mayApprove && !mayApproveAtLowerLevels && !mayAcceptAtLowerLevels) {
return new ArrayList<>();
}
int lowestNumberOrgUnitLevel = getCurrentUsersLowestNumberOrgUnitLevel();
boolean canSeeAllDimensions =
CollectionUtils.isEmpty(userService.getCoDimensionConstraints(userCredentials))
&& CollectionUtils.isEmpty(userService.getCogDimensionConstraints(userCredentials));
List<DataApprovalLevel> approvalLevels = getAllDataApprovalLevels();
List<DataApprovalLevel> userDataApprovalLevels = new ArrayList<>();
boolean addLevel = false;
for (DataApprovalLevel approvalLevel : approvalLevels) {
if (!addLevel && approvalLevel.getOrgUnitLevel() >= lowestNumberOrgUnitLevel) {
CategoryOptionGroupSet cogs = approvalLevel.getCategoryOptionGroupSet();
addLevel =
securityService.canRead(approvalLevel) && cogs == null
? canSeeAllDimensions
: (securityService.canRead(cogs)
&& !CollectionUtils.isEmpty(categoryService.getCategoryOptionGroups(cogs)));
}
if (addLevel) {
userDataApprovalLevels.add(approvalLevel);
}
}
return userDataApprovalLevels;
}
protected void addDataValue(
OrganisationUnit unit,
Period period,
String expression,
String value,
Set<DataValue> oldList,
Set<DataValue> newList) {
// value = value.replaceAll( "\\.", "" ).replace( ",", "." );
DataElementOperand operand =
expressionService.getOperandsInExpression(expression).iterator().next();
DataElement dataElement = dataElementService.getDataElement(operand.getDataElementId());
DataElementCategoryOptionCombo optionCombo =
categoryService.getDataElementCategoryOptionCombo(operand.getOptionComboId());
String storedBy = currentUserService.getCurrentUsername();
DataValue dataValue = dataValueService.getDataValue(unit, dataElement, period, optionCombo);
if (dataValue == null) {
dataValue =
new DataValue(dataElement, period, unit, value, storedBy, new Date(), null, optionCombo);
dataValueService.addDataValue(dataValue);
newList.add(dataValue);
} else {
DataValue backedUpDataValue =
new DataValue(dataElement, period, unit, dataValue.getValue(), optionCombo);
oldList.add(backedUpDataValue);
dataValue.setValue(value);
dataValue.setTimestamp(new Date());
dataValue.setStoredBy(storedBy);
dataValueService.updateDataValue(dataValue);
}
}
public String execute() {
DataElement dataElement = dataElementService.getDataElement(dataElementId);
DataElementCategoryOptionCombo categoryOptionCombo =
categoryService.getDataElementCategoryOptionCombo(categoryOptionComboId);
Period period = PeriodType.createPeriodExternalId(periodId);
OrganisationUnit source = organisationUnitService.getOrganisationUnit(organisationUnitId);
DataValue dataValue =
dataValueService.getDataValue(source, dataElement, period, categoryOptionCombo);
boolean isMarked = dataValue.isFollowup();
dataValue.setFollowup(!isMarked);
dataValueService.updateDataValue(dataValue);
message = !isMarked ? "marked" : "unmarked";
log.info(!isMarked ? "Data value marked for follow-up" : "Data value unmarked for follow-up");
return SUCCESS;
}
@Override
public String execute() {
optionCombo = categoryService.getDataElementCategoryOptionCombo(id);
return SUCCESS;
}
@Override
protected DataElementCategoryOption getMatching(DataElementCategoryOption object) {
return categoryService.getDataElementCategoryOptionByName(object.getName());
}
@RequestMapping(method = RequestMethod.POST, produces = "text/plain")
public void saveDataValue(
@RequestParam String de,
@RequestParam String cc,
@RequestParam String pe,
@RequestParam String ou,
@RequestParam String value,
HttpServletResponse response) {
DataElement dataElement = dataElementService.getDataElement(de);
if (dataElement == null) {
ContextUtils.conflictResponse(response, "Illegal data element identifier: " + de);
return;
}
DataElementCategoryOptionCombo categoryOptionCombo =
categoryService.getDataElementCategoryOptionCombo(cc);
if (categoryOptionCombo == null) {
ContextUtils.conflictResponse(response, "Illegal category option combo identifier: " + cc);
return;
}
Period period = PeriodType.getPeriodFromIsoString(pe);
if (period == null) {
ContextUtils.conflictResponse(response, "Illegal period identifier: " + pe);
return;
}
OrganisationUnit organisationUnit = organisationUnitService.getOrganisationUnit(ou);
if (organisationUnit == null) {
ContextUtils.conflictResponse(response, "Illegal organisation unit identifier: " + ou);
return;
}
if (dataSetService.isLocked(dataElement, period, organisationUnit, null)) {
ContextUtils.conflictResponse(response, "Data set is locked");
return;
}
value = StringUtils.trimToNull(value);
String storedBy = currentUserService.getCurrentUsername();
Date now = new Date();
DataValue dataValue =
dataValueService.getDataValue(organisationUnit, dataElement, period, categoryOptionCombo);
if (dataValue == null) {
if (value != null) {
dataValue =
new DataValue(
dataElement,
period,
organisationUnit,
value,
storedBy,
now,
null,
categoryOptionCombo);
dataValueService.addDataValue(dataValue);
}
} else {
dataValue.setValue(value);
dataValue.setTimestamp(now);
dataValue.setStoredBy(storedBy);
dataValueService.updateDataValue(dataValue);
}
}
@Override
public String execute() throws Exception {
for (String aggregateValue : aggregateValues) {
// -----------------------------------------------------------------
// Get params
// -----------------------------------------------------------------
String[] info = aggregateValue.split(SEPERATE_SIGN);
int dataElementId = Integer.parseInt(info[0]);
int optionComboId = Integer.parseInt(info[1]);
String periodIsoId = info[2];
int orgunitId = Integer.parseInt(info[3]);
String resultValue = info[4];
// -----------------------------------------------------------------
// Create objects
// -----------------------------------------------------------------
DataElement dataElement = dataElementService.getDataElement(dataElementId);
DataElementCategoryOptionCombo optionCombo =
categoryService.getDataElementCategoryOptionCombo(optionComboId);
DataElementCategoryOptionCombo attributeOptioncombo =
categoryService.getDefaultDataElementCategoryOptionCombo();
Period period = PeriodType.getPeriodFromIsoString(periodIsoId);
OrganisationUnit orgunit = organisationUnitService.getOrganisationUnit(orgunitId);
DataValue dataValue =
dataValueService.getDataValue(
dataElement, period, orgunit, optionCombo, attributeOptioncombo);
// -----------------------------------------------------------------
// Save/Update/Delete data-values
// -----------------------------------------------------------------
if (resultValue != "0.0") {
if (dataValue == null) {
dataValue =
new DataValue(
dataElement,
period,
orgunit,
optionCombo,
attributeOptioncombo,
"" + resultValue,
CaseAggregationCondition.AUTO_STORED_BY,
new Date(),
null);
dataValueService.addDataValue(dataValue);
} else {
dataValue.setValue(resultValue);
dataValue.setLastUpdated(new Date());
dataValue.setStoredBy(CaseAggregationCondition.AUTO_STORED_BY);
dataValueService.updateDataValue(dataValue);
}
} else if (dataValue != null) {
dataValueService.deleteDataValue(dataValue);
}
}
return SUCCESS;
}
@Transactional
public void export(
Collection<Integer> dataElementIds,
Collection<Integer> indicatorIds,
Collection<Integer> periodIds,
Collection<Integer> organisationUnitIds,
Collection<Integer> organisationUnitGroupIds,
TaskId id) {
final int cpuCores = SystemUtils.getCpuCores();
Clock clock =
new Clock()
.startClock()
.logTime(
"Data mart export process started, number of CPU cores: "
+ cpuCores
+ ", "
+ SystemUtils.getMemoryString());
notifier.clear(id, DATAMART).notify(id, DATAMART, "Data mart export process started");
// ---------------------------------------------------------------------
// Recreate temporary tables
// ---------------------------------------------------------------------
dataMartManager.dropTempAggregatedTables();
dataMartManager.createTempAggregatedTables();
clock.logTime("Recreated temporary tables");
// ---------------------------------------------------------------------
// Replace null with empty collection
// ---------------------------------------------------------------------
dataElementIds = dataElementIds != null ? dataElementIds : new ArrayList<Integer>();
indicatorIds = indicatorIds != null ? indicatorIds : new ArrayList<Integer>();
periodIds = periodIds != null ? periodIds : new ArrayList<Integer>();
organisationUnitIds =
organisationUnitIds != null ? organisationUnitIds : new ArrayList<Integer>();
organisationUnitGroupIds =
organisationUnitGroupIds != null ? organisationUnitGroupIds : new ArrayList<Integer>();
clock.logTime(
"Data elements: "
+ dataElementIds.size()
+ ", indicators: "
+ indicatorIds.size()
+ ", periods: "
+ periodIds.size()
+ ", org units: "
+ organisationUnitIds.size());
// ---------------------------------------------------------------------
// Get objects
// ---------------------------------------------------------------------
final Collection<Indicator> indicators = indicatorService.getIndicators(indicatorIds);
final Collection<Period> periods = periodService.getPeriods(periodIds);
final List<OrganisationUnit> organisationUnits =
new ArrayList<OrganisationUnit>(
organisationUnitService.getOrganisationUnits(organisationUnitIds));
final Collection<OrganisationUnitGroup> organisationUnitGroups =
organisationUnitGroupService.getOrganisationUnitGroups(organisationUnitGroupIds);
final Collection<DataElement> dataElements = dataElementService.getDataElements(dataElementIds);
clock.logTime("Retrieved meta-data objects");
notifier.notify(id, DATAMART, "Filtering meta-data");
// ---------------------------------------------------------------------
// Filter objects
// ---------------------------------------------------------------------
organisationUnitService.filterOrganisationUnitsWithoutData(organisationUnits);
FilterUtils.filter(dataElements, new AggregatableDataElementFilter());
FilterUtils.filter(dataElements, new DataElementWithAggregationFilter());
expressionService.filterInvalidIndicators(indicators);
clock.logTime("Filtered objects");
notifier.notify(id, DATAMART, "Loading indicators");
// ---------------------------------------------------------------------
// Explode indicator expressions
// ---------------------------------------------------------------------
for (Indicator indicator : indicators) {
indicator.setExplodedNumerator(expressionService.explodeExpression(indicator.getNumerator()));
indicator.setExplodedDenominator(
expressionService.explodeExpression(indicator.getDenominator()));
}
clock.logTime("Exploded indicator expressions");
notifier.notify(id, DATAMART, "Loading data elements");
// ---------------------------------------------------------------------
// Get operands
// ---------------------------------------------------------------------
final Collection<DataElementOperand> dataElementOperands =
categoryService.getOperands(dataElements);
final List<DataElementOperand> indicatorOperands =
new ArrayList<DataElementOperand>(
categoryService.populateOperands(
expressionService.getOperandsInIndicators(indicators)));
Set<DataElementOperand> allOperands = new HashSet<DataElementOperand>();
allOperands.addAll(dataElementOperands);
allOperands.addAll(indicatorOperands);
clock.logTime("Retrieved operands: " + allOperands.size());
notifier.notify(id, DATAMART, "Loading periods");
// ---------------------------------------------------------------------
// Filter out future periods
// ---------------------------------------------------------------------
FilterUtils.filter(periods, new PastAndCurrentPeriodFilter());
clock.logTime("Number of periods: " + periods.size());
notifier.notify(id, DATAMART, "Filtering data elements without data");
// ---------------------------------------------------------------------
// Remove operands without data
// ---------------------------------------------------------------------
allOperands = dataMartManager.getOperandsWithData(allOperands);
indicatorOperands.retainAll(allOperands);
clock.logTime(
"Number of operands with data: "
+ allOperands.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Populating crosstabulation table");
// ---------------------------------------------------------------------
// Create crosstabtable
// ---------------------------------------------------------------------
final Collection<Integer> intersectingPeriodIds =
ConversionUtils.getIdentifiers(Period.class, periodService.getIntersectionPeriods(periods));
final PeriodHierarchy periodHierarchy = periodService.getPeriodHierarchy(periods);
final Set<Integer> orgUnitChildrenIds =
organisationUnitService.getOrganisationUnitHierarchy().getChildren(organisationUnitIds);
final List<Integer> crossTabOrgUnitIds = new ArrayList<Integer>(orgUnitChildrenIds);
final String key = crossTabService.createCrossTabTable(crossTabOrgUnitIds);
final List<DataElementOperand> operandList = new ArrayList<DataElementOperand>(allOperands);
Collections.shuffle(operandList);
final List<List<DataElementOperand>> operandPages =
new PaginatedList<DataElementOperand>(operandList).setNumberOfPages(cpuCores).getPages();
List<Future<?>> crossTabFutures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
crossTabFutures.add(
crossTabService.populateCrossTabTable(
operandPage, intersectingPeriodIds, crossTabOrgUnitIds, key));
}
ConcurrentUtils.waitForCompletion(crossTabFutures);
clock.logTime("Populated crosstab table, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Exporting data element data");
final boolean isDataElements = true;
final boolean isIndicators = indicators != null && indicators.size() > 0;
final int groupLevel =
(Integer)
systemSettingManager.getSystemSetting(
KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL);
final boolean isGroups =
organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0;
if (isDataElements) {
// -----------------------------------------------------------------
// 1. Export data element values
// -----------------------------------------------------------------
if (operandList.size() > 0) {
final OrganisationUnitHierarchy orgUnitHierarchy =
organisationUnitService
.getOrganisationUnitHierarchy()
.prepareChildren(organisationUnits);
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
futures.add(
dataElementDataMart.exportDataValues(
operandPage,
periods,
organisationUnits,
null,
periodHierarchy,
orgUnitHierarchy,
AggregatedDataValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
}
clock.logTime(
"Exported values for data element operands ("
+ operandList.size()
+ "), pages: "
+ operandPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping data element index");
// -----------------------------------------------------------------
// 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropDataValueIndex();
clock.logTime("Dropped data element index");
notifier.notify(id, DATAMART, "Deleting existing data element data");
// -----------------------------------------------------------------
// 3. Delete existing aggregated data values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedDataValues(periodIds);
clock.logTime("Deleted existing data element data");
notifier.notify(id, DATAMART, "Copying data element data from temporary table");
// -----------------------------------------------------------------
// 4. Copy aggregated data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedDataValuesFromTemp();
clock.logTime("Copied data element data from temporary table");
notifier.notify(id, DATAMART, "Creating data element index");
// -----------------------------------------------------------------
// 5. Create data element index
// -----------------------------------------------------------------
dataMartManager.createDataValueIndex();
clock.logTime("Created data element index");
}
if (isGroups && isDataElements) {
// -----------------------------------------------------------------
// 1. Export data element values
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Exporting data element org unit data");
Collection<OrganisationUnit> groupOrganisationUnits =
new HashSet<OrganisationUnit>(organisationUnits);
FilterUtils.filter(
groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter(groupLevel));
if (operandList.size() > 0) {
final OrganisationUnitHierarchy orgUnitHierarchy =
organisationUnitService
.getOrganisationUnitHierarchy()
.prepareChildren(groupOrganisationUnits, organisationUnitGroups);
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<DataElementOperand> operandPage : operandPages) {
futures.add(
dataElementDataMart.exportDataValues(
operandPage,
periods,
groupOrganisationUnits,
organisationUnitGroups,
periodHierarchy,
orgUnitHierarchy,
AggregatedOrgUnitDataValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
}
clock.logTime(
"Exported values for data element operands ("
+ operandList.size()
+ "), pages: "
+ operandPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping data element data indexes");
// -----------------------------------------------------------------
// 2. Drop data element index
// -----------------------------------------------------------------
dataMartManager.dropOrgUnitDataValueIndex();
clock.logTime("Dropped org unit data element index");
notifier.notify(id, DATAMART, "Deleting existing org unit data element data");
// -----------------------------------------------------------------
// 3. Delete existing aggregated data values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitDataValues(periodIds);
clock.logTime("Deleted existing aggregated org unit datavalues");
notifier.notify(id, DATAMART, "Copying org unit data element data");
// -----------------------------------------------------------------
// 4. Copy aggregated org unit data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp();
clock.logTime("Copied org unit data element data from temporary table");
notifier.notify(id, DATAMART, "Creating org unit data element index");
// -----------------------------------------------------------------
// 5. Create org unit data element index
// -----------------------------------------------------------------
dataMartManager.createOrgUnitDataValueIndex();
clock.logTime("Created org unit data element index");
notifier.notify(id, DATAMART, "Exporting data for org unit indicator data");
}
crossTabService.dropCrossTabTable(key);
List<List<OrganisationUnit>> organisationUnitPages =
new PaginatedList<OrganisationUnit>(organisationUnits)
.setNumberOfPages(cpuCores)
.getPages();
if (isIndicators) {
// -----------------------------------------------------------------
// 1. Create and populate aggregated data cache
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Populating aggregated data cache");
crossTabService.createAggregatedDataCache(indicatorOperands, key);
List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
aggregatedDataCacheFutures.add(
crossTabService.populateAggregatedDataCache(
indicatorOperands, periods, organisationUnitPage, key));
}
ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);
clock.logTime(
"Created aggregated data cache, number of indicator operands: "
+ indicatorOperands.size());
notifier.notify(id, DATAMART, "Exporting indicator data");
// -----------------------------------------------------------------
// 2. Export indicator values
// -----------------------------------------------------------------
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
futures.add(
indicatorDataMart.exportIndicatorValues(
indicators,
periods,
organisationUnitPage,
null,
indicatorOperands,
AggregatedIndicatorValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
clock.logTime(
"Exported values for indicators ("
+ indicators.size()
+ "), pages: "
+ organisationUnitPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping indicator index");
// -----------------------------------------------------------------
// 3. Drop aggregated data cache and indicator index
// -----------------------------------------------------------------
crossTabService.dropAggregatedDataCache(key);
dataMartManager.dropIndicatorValueIndex();
clock.logTime("Dropped indicator index, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Deleting existing indicator data");
// -----------------------------------------------------------------
// 4. Delete existing aggregated indicator values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedIndicatorValues(periodIds);
clock.logTime("Deleted existing indicator data");
notifier.notify(id, DATAMART, "Copying indicator data from temporary table");
// -----------------------------------------------------------------
// 5. Copy aggregated data values from temporary table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedIndicatorValuesFromTemp();
clock.logTime("Copied indicator data from temporary table");
notifier.notify(id, DATAMART, "Creating indicator index");
// -----------------------------------------------------------------
// 6. Create indicator index
// -----------------------------------------------------------------
dataMartManager.createIndicatorValueIndex();
clock.logTime("Created indicator index");
}
if (isGroups && isIndicators) {
// -----------------------------------------------------------------
// 1. Create aggregated data cache
// -----------------------------------------------------------------
notifier.notify(id, DATAMART, "Populating aggregated data cache");
crossTabService.createAggregatedOrgUnitDataCache(indicatorOperands, key);
List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
aggregatedDataCacheFutures.add(
crossTabService.populateAggregatedOrgUnitDataCache(
indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key));
}
ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);
clock.logTime("Created aggregated org unit data cache");
notifier.notify(id, DATAMART, "Exporting org unit indicator data");
// -----------------------------------------------------------------
// 2. Export indicator values
// -----------------------------------------------------------------
List<Future<?>> futures = new ArrayList<Future<?>>();
for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
futures.add(
indicatorDataMart.exportIndicatorValues(
indicators,
periods,
organisationUnitPage,
organisationUnitGroups,
indicatorOperands,
AggregatedOrgUnitIndicatorValueTempBatchHandler.class,
key));
}
ConcurrentUtils.waitForCompletion(futures);
clock.logTime(
"Exported values for indicators ("
+ indicators.size()
+ "), pages: "
+ organisationUnitPages.size()
+ ", "
+ SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Dropping org unit indicator index");
// -----------------------------------------------------------------
// 3. Drop aggregated data cache and indicator index
// -----------------------------------------------------------------
crossTabService.dropAggregatedOrgUnitDataCache(key);
dataMartManager.dropOrgUnitIndicatorValueIndex();
clock.logTime("Dropped org unit indicator index, " + SystemUtils.getMemoryString());
notifier.notify(id, DATAMART, "Deleting existing org unit indicator data");
// -----------------------------------------------------------------
// 4. Delete existing aggregated indicator values
// -----------------------------------------------------------------
dataMartManager.deleteAggregatedOrgUnitIndicatorValues(periodIds);
clock.logTime("Deleted existing aggregated org unit indicatorvalues");
notifier.notify(id, DATAMART, "Copying org unit indicator data from temporary table");
// -----------------------------------------------------------------
// 5. Copy aggregated org unit indicator values from temp table
// -----------------------------------------------------------------
dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();
clock.logTime("Copied org unit indicator data from temporary table");
notifier.notify(id, DATAMART, "Creating org unit indicator indexes");
// -----------------------------------------------------------------
// 6. Create org unit indicator index
// -----------------------------------------------------------------
dataMartManager.createOrgUnitIndicatorValueIndex();
clock.logTime("Created org unit indicator index");
}
// ---------------------------------------------------------------------
// Drop temporary tables
// ---------------------------------------------------------------------
dataMartManager.dropTempAggregatedTables();
clock.logTime("Dropped crosstab table");
clock.logTime("Data mart export process completed");
notifier.notify(id, DATAMART, INFO, "Data mart process completed", true);
}
@Override
public void setUpTest() {
mockDataValueBatchHandler = new MockBatchHandler<>();
mockBatchHandlerFactory = new MockBatchHandlerFactory();
mockBatchHandlerFactory.registerBatchHandler(
DataValueBatchHandler.class, mockDataValueBatchHandler);
setDependency(dataValueSetService, "batchHandlerFactory", mockBatchHandlerFactory);
categoryOptionA = createCategoryOption('A');
categoryOptionB = createCategoryOption('B');
categoryA = createDataElementCategory('A', categoryOptionA, categoryOptionB);
categoryComboA = createCategoryCombo('A', categoryA);
ocDef = categoryService.getDefaultDataElementCategoryOptionCombo();
ocA = createCategoryOptionCombo(categoryComboA, categoryOptionA);
ocB = createCategoryOptionCombo(categoryComboA, categoryOptionB);
deA = createDataElement('A');
deB = createDataElement('B');
deC = createDataElement('C');
deD = createDataElement('D');
dsA = createDataSet('A', new MonthlyPeriodType());
ouA = createOrganisationUnit('A');
ouB = createOrganisationUnit('B');
ouC = createOrganisationUnit('C');
peA =
createPeriod(
PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME),
getDate(2012, 1, 1),
getDate(2012, 1, 31));
peB =
createPeriod(
PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME),
getDate(2012, 2, 1),
getDate(2012, 2, 29));
ocA.setUid("kjuiHgy67hg");
ocB.setUid("Gad33qy67g5");
deA.setUid("f7n9E0hX8qk");
deB.setUid("Ix2HsbDMLea");
deC.setUid("eY5ehpbEsB7");
dsA.setUid("pBOMPrpg1QX");
ouA.setUid("DiszpKrYNg8");
ouB.setUid("BdfsJfj87js");
ouC.setUid("j7Hg26FpoIa");
ocA.setCode("OC_A");
ocB.setCode("OC_B");
deA.setCode("DE_A");
deB.setCode("DE_B");
deC.setCode("DE_C");
deD.setCode("DE_D");
dsA.setCode("DS_A");
ouA.setCode("OU_A");
ouB.setCode("OU_B");
ouC.setCode("OU_C");
categoryService.addDataElementCategoryOption(categoryOptionA);
categoryService.addDataElementCategoryOption(categoryOptionB);
categoryService.addDataElementCategory(categoryA);
categoryService.addDataElementCategoryCombo(categoryComboA);
categoryService.addDataElementCategoryOptionCombo(ocA);
categoryService.addDataElementCategoryOptionCombo(ocB);
dataElementService.addDataElement(deA);
dataElementService.addDataElement(deB);
dataElementService.addDataElement(deC);
dataElementService.addDataElement(deD);
dataSetService.addDataSet(dsA);
organisationUnitService.addOrganisationUnit(ouA);
organisationUnitService.addOrganisationUnit(ouB);
organisationUnitService.addOrganisationUnit(ouC);
periodService.addPeriod(peA);
periodService.addPeriod(peB);
user = createUser('A');
user.setOrganisationUnits(Sets.newHashSet(ouA, ouB));
CurrentUserService currentUserService = new MockCurrentUserService(user);
setDependency(dataValueSetService, "currentUserService", currentUserService);
}