@Override
  public String execute() throws Exception {
    CategoryOptionGroupSet categoryOptionGroupSet =
        dataElementCategoryService.getCategoryOptionGroupSet(id);

    dataElementCategoryService.deleteCategoryOptionGroupSet(categoryOptionGroupSet);

    return SUCCESS;
  }
  @Override
  public void setUpTest() {
    categoryCombo =
        categoryService.getDataElementCategoryComboByName(
            DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME);

    categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo();

    dataElementIds = new HashSet<>();
    indicatorIds = new HashSet<>();
    periodIds = new HashSet<>();
    organisationUnitIds = new HashSet<>();
  }
  @Override
  public DataApprovalLevel getLowestDataApprovalLevel(
      OrganisationUnit orgUnit, DataElementCategoryOptionCombo attributeOptionCombo) {
    Set<CategoryOptionGroupSet> cogSets = null;

    if (attributeOptionCombo != null
        && attributeOptionCombo != categoryService.getDefaultDataElementCategoryOptionCombo()) {
      cogSets = new HashSet<>();

      for (DataElementCategoryOption option : attributeOptionCombo.getCategoryOptions()) {
        if (option.getGroupSets() != null) {
          cogSets.addAll(option.getGroupSets());
        }
      }
    }

    int orgUnitLevel = orgUnit.getLevel();

    List<DataApprovalLevel> approvalLevels = getDataApprovalLevelsByOrgUnitLevel(orgUnitLevel);

    for (DataApprovalLevel level : Lists.reverse(approvalLevels)) {
      if (level.getCategoryOptionGroupSet() == null) {
        if (cogSets == null) {
          return level;
        }
      } else if (cogSets != null && cogSets.contains(level.getCategoryOptionGroupSet())) {
        return level;
      }
    }

    return null;
  }
  public String execute() {
    if (dataSetId != null) {
      dataSet = dataSetService.getDataSet(dataSetId);

      operands =
          new ArrayList<DataElementOperand>(
              dataElementCategoryService.getFullOperands(dataSet.getDataElements()));
    } else {
      operands = new ArrayList<DataElementOperand>();
    }

    if (key != null) {
      Iterator<DataElementOperand> iterator = operands.iterator();

      while (iterator.hasNext()) {
        DataElementOperand operand = iterator.next();

        if (operand.getOperandName().toLowerCase().indexOf(key.toLowerCase()) == -1) {
          iterator.remove();
        }
      }
    }

    Collections.sort(operands, new DataElementOperandNameComparator());

    if (usePaging) {
      this.paging = createPaging(operands.size());

      operands = operands.subList(paging.getStartPos(), paging.getEndPos());
    }

    return SUCCESS;
  }
  public String execute() {
    try {
      DataElementCategory category = dataElementCategoryService.getDataElementCategory(id);

      dataElementCategoryService.deleteDataElementCategory(category);
    } catch (DeleteNotAllowedException ex) {
      if (ex.getErrorCode().equals(DeleteNotAllowedException.ERROR_ASSOCIATED_BY_OTHER_OBJECTS)) {
        message =
            i18n.getString("object_not_deleted_associated_by_objects") + " " + ex.getMessage();

        return ERROR;
      }
    }

    return SUCCESS;
  }
  @Override
  protected void importMatching(DataElementCategoryOption object, DataElementCategoryOption match) {
    match.setCode(object.getCode());
    match.setUid(object.getUid());
    match.setName(object.getName());

    categoryService.updateDataElementCategoryOption(match);
  }
  @Override
  public String execute() {
    dataElementCategories =
        new ArrayList<>(dataElementCategoryService.getAllDataElementCategories());

    Collections.sort(dataElementCategories, IdentifiableObjectNameComparator.INSTANCE);

    return SUCCESS;
  }
  /**
   * Can the user read from this CategoryOptionGroupSet (COGS)?
   *
   * <p>If the COGS is null, then the user must have no dimension constraints. (In other words, the
   * user must be able to read across all category option groups.)
   *
   * <p>If the COGS is not null, then the user must be able to read at least one category option
   * group from the category option group set.
   *
   * @param cogs The category option group set to test
   * @return true if user can read at least one category option group.
   */
  private boolean canReadCOGS(User user, CategoryOptionGroupSet cogs) {
    if (cogs == null) {
      UserCredentials userCredentials = user.getUserCredentials();

      return CollectionUtils.isEmpty(userCredentials.getCogsDimensionConstraints())
          && CollectionUtils.isEmpty(userCredentials.getCatDimensionConstraints());
    }

    return !CollectionUtils.isEmpty(categoryService.getCategoryOptionGroups(cogs));
  }
  public String execute() throws Exception {
    // ---------------------------------------------------------------------
    // Prepare values
    // ---------------------------------------------------------------------

    code = nullIfEmpty(code);
    shortName = nullIfEmpty(shortName);
    description = nullIfEmpty(description);

    PeriodType periodType = PeriodType.getPeriodTypeByName(frequencySelect);

    DataSet dataSet = new DataSet(name, shortName, code, periodType);

    MapLegendSet legendSet = mappingService.getMapLegendSet(selectedLegendSetId);

    dataSet.setExpiryDays(expiryDays);
    dataSet.setTimelyDays(timelyDays);
    dataSet.setSkipAggregation(skipAggregation);

    for (String id : dataElementsSelectedList) {
      dataSet.addDataElement(dataElementService.getDataElement(Integer.parseInt(id)));
    }

    Set<Indicator> indicators = new HashSet<Indicator>();

    for (String id : indicatorsSelectedList) {
      indicators.add(indicatorService.getIndicator(Integer.parseInt(id)));
    }

    if (categoryComboId != null) {
      dataSet.setCategoryCombo(categoryService.getDataElementCategoryCombo(categoryComboId));
    }

    dataSet.setDescription(description);
    dataSet.setVersion(1);
    dataSet.setMobile(false);
    dataSet.setIndicators(indicators);
    dataSet.setNotificationRecipients(userGroupService.getUserGroup(notificationRecipients));
    dataSet.setAllowFuturePeriods(allowFuturePeriods);
    dataSet.setFieldCombinationRequired(fieldCombinationRequired);
    dataSet.setValidCompleteOnly(validCompleteOnly);
    dataSet.setNotifyCompletingUser(notifyCompletingUser);
    dataSet.setApproveData(approveData);
    dataSet.setSkipOffline(skipOffline);
    dataSet.setDataElementDecoration(dataElementDecoration);
    dataSet.setRenderAsTabs(renderAsTabs);
    dataSet.setRenderHorizontally(renderHorizontally);
    dataSet.setLegendSet(legendSet);

    dataSetService.addDataSet(dataSet);

    userService.assignDataSetToUserRole(dataSet);

    return SUCCESS;
  }
  @Override
  public String execute() {
    DataElementCategoryCombo categoryCombo =
        dataElementCategoryService.getDataElementCategoryCombo(id);

    DataElementCategoryCombo defaultCategoryCombo =
        dataElementCategoryService.getDataElementCategoryComboByName(
            DataElementCategoryCombo.DEFAULT_CATEGORY_COMBO_NAME);

    if (!categoryCombo.equals(defaultCategoryCombo)) {
      try {
        dataElementCategoryService.deleteDataElementCategoryCombo(categoryCombo);
      } catch (DeleteNotAllowedException ex) {
        if (ex.getErrorCode().equals(DeleteNotAllowedException.ERROR_ASSOCIATED_BY_OTHER_OBJECTS)) {
          message =
              i18n.getString("object_not_deleted_associated_by_objects") + " " + ex.getMessage();
          return ERROR;
        }
      }
    }

    return SUCCESS;
  }
Beispiel #11
0
  @Override
  public Set<CategoryOptionGroup> getCogDimensionConstraints(UserCredentials userCredentials) {
    Set<CategoryOptionGroup> groups = null;

    Set<CategoryOptionGroupSet> cogsConstraints = userCredentials.getCogsDimensionConstraints();

    if (cogsConstraints != null && !cogsConstraints.isEmpty()) {
      groups = new HashSet<>();

      for (CategoryOptionGroupSet cogs : cogsConstraints) {
        groups.addAll(categoryService.getCategoryOptionGroups(cogs));
      }
    }

    return groups;
  }
Beispiel #12
0
  @Override
  public Set<DataElementCategoryOption> getCoDimensionConstraints(UserCredentials userCredentials) {
    Set<DataElementCategoryOption> options = null;

    Set<DataElementCategory> catConstraints = userCredentials.getCatDimensionConstraints();

    if (catConstraints != null && !catConstraints.isEmpty()) {
      options = new HashSet<>();

      for (DataElementCategory category : catConstraints) {
        options.addAll(categoryService.getDataElementCategoryOptions(category));
      }
    }

    return options;
  }
  public String execute() {
    if (name != null) {
      DataElementCategoryOption match =
          dataElementCategoryService.getDataElementCategoryOptionByName(name);

      if (match != null && (id == null || match.getId() != id)) {
        message = i18n.getString("name_in_use");

        return ERROR;
      }
    }

    message = "ok";

    return SUCCESS;
  }
  @Override
  public List<DataApprovalLevel> getUserDataApprovalLevels() {
    UserCredentials userCredentials = currentUserService.getCurrentUser().getUserCredentials();

    boolean mayApprove = userCredentials.isAuthorized(DataApproval.AUTH_APPROVE);
    boolean mayApproveAtLowerLevels =
        userCredentials.isAuthorized(DataApproval.AUTH_APPROVE_LOWER_LEVELS);
    boolean mayAcceptAtLowerLevels =
        userCredentials.isAuthorized(DataApproval.AUTH_ACCEPT_LOWER_LEVELS);

    if (!mayApprove && !mayApproveAtLowerLevels && !mayAcceptAtLowerLevels) {
      return new ArrayList<>();
    }

    int lowestNumberOrgUnitLevel = getCurrentUsersLowestNumberOrgUnitLevel();

    boolean canSeeAllDimensions =
        CollectionUtils.isEmpty(userService.getCoDimensionConstraints(userCredentials))
            && CollectionUtils.isEmpty(userService.getCogDimensionConstraints(userCredentials));

    List<DataApprovalLevel> approvalLevels = getAllDataApprovalLevels();
    List<DataApprovalLevel> userDataApprovalLevels = new ArrayList<>();

    boolean addLevel = false;

    for (DataApprovalLevel approvalLevel : approvalLevels) {
      if (!addLevel && approvalLevel.getOrgUnitLevel() >= lowestNumberOrgUnitLevel) {
        CategoryOptionGroupSet cogs = approvalLevel.getCategoryOptionGroupSet();

        addLevel =
            securityService.canRead(approvalLevel) && cogs == null
                ? canSeeAllDimensions
                : (securityService.canRead(cogs)
                    && !CollectionUtils.isEmpty(categoryService.getCategoryOptionGroups(cogs)));
      }

      if (addLevel) {
        userDataApprovalLevels.add(approvalLevel);
      }
    }

    return userDataApprovalLevels;
  }
  protected void addDataValue(
      OrganisationUnit unit,
      Period period,
      String expression,
      String value,
      Set<DataValue> oldList,
      Set<DataValue> newList) {
    // value = value.replaceAll( "\\.", "" ).replace( ",", "." );

    DataElementOperand operand =
        expressionService.getOperandsInExpression(expression).iterator().next();

    DataElement dataElement = dataElementService.getDataElement(operand.getDataElementId());

    DataElementCategoryOptionCombo optionCombo =
        categoryService.getDataElementCategoryOptionCombo(operand.getOptionComboId());

    String storedBy = currentUserService.getCurrentUsername();

    DataValue dataValue = dataValueService.getDataValue(unit, dataElement, period, optionCombo);

    if (dataValue == null) {
      dataValue =
          new DataValue(dataElement, period, unit, value, storedBy, new Date(), null, optionCombo);
      dataValueService.addDataValue(dataValue);

      newList.add(dataValue);
    } else {
      DataValue backedUpDataValue =
          new DataValue(dataElement, period, unit, dataValue.getValue(), optionCombo);

      oldList.add(backedUpDataValue);

      dataValue.setValue(value);
      dataValue.setTimestamp(new Date());
      dataValue.setStoredBy(storedBy);

      dataValueService.updateDataValue(dataValue);
    }
  }
  public String execute() {
    DataElement dataElement = dataElementService.getDataElement(dataElementId);
    DataElementCategoryOptionCombo categoryOptionCombo =
        categoryService.getDataElementCategoryOptionCombo(categoryOptionComboId);
    Period period = PeriodType.createPeriodExternalId(periodId);
    OrganisationUnit source = organisationUnitService.getOrganisationUnit(organisationUnitId);

    DataValue dataValue =
        dataValueService.getDataValue(source, dataElement, period, categoryOptionCombo);

    boolean isMarked = dataValue.isFollowup();

    dataValue.setFollowup(!isMarked);

    dataValueService.updateDataValue(dataValue);

    message = !isMarked ? "marked" : "unmarked";

    log.info(!isMarked ? "Data value marked for follow-up" : "Data value unmarked for follow-up");

    return SUCCESS;
  }
  @Override
  public String execute() {
    optionCombo = categoryService.getDataElementCategoryOptionCombo(id);

    return SUCCESS;
  }
 @Override
 protected DataElementCategoryOption getMatching(DataElementCategoryOption object) {
   return categoryService.getDataElementCategoryOptionByName(object.getName());
 }
  @RequestMapping(method = RequestMethod.POST, produces = "text/plain")
  public void saveDataValue(
      @RequestParam String de,
      @RequestParam String cc,
      @RequestParam String pe,
      @RequestParam String ou,
      @RequestParam String value,
      HttpServletResponse response) {
    DataElement dataElement = dataElementService.getDataElement(de);

    if (dataElement == null) {
      ContextUtils.conflictResponse(response, "Illegal data element identifier: " + de);
      return;
    }

    DataElementCategoryOptionCombo categoryOptionCombo =
        categoryService.getDataElementCategoryOptionCombo(cc);

    if (categoryOptionCombo == null) {
      ContextUtils.conflictResponse(response, "Illegal category option combo identifier: " + cc);
      return;
    }

    Period period = PeriodType.getPeriodFromIsoString(pe);

    if (period == null) {
      ContextUtils.conflictResponse(response, "Illegal period identifier: " + pe);
      return;
    }

    OrganisationUnit organisationUnit = organisationUnitService.getOrganisationUnit(ou);

    if (organisationUnit == null) {
      ContextUtils.conflictResponse(response, "Illegal organisation unit identifier: " + ou);
      return;
    }

    if (dataSetService.isLocked(dataElement, period, organisationUnit, null)) {
      ContextUtils.conflictResponse(response, "Data set is locked");
      return;
    }

    value = StringUtils.trimToNull(value);

    String storedBy = currentUserService.getCurrentUsername();

    Date now = new Date();

    DataValue dataValue =
        dataValueService.getDataValue(organisationUnit, dataElement, period, categoryOptionCombo);

    if (dataValue == null) {
      if (value != null) {
        dataValue =
            new DataValue(
                dataElement,
                period,
                organisationUnit,
                value,
                storedBy,
                now,
                null,
                categoryOptionCombo);
        dataValueService.addDataValue(dataValue);
      }
    } else {
      dataValue.setValue(value);
      dataValue.setTimestamp(now);
      dataValue.setStoredBy(storedBy);

      dataValueService.updateDataValue(dataValue);
    }
  }
  @Override
  public String execute() throws Exception {
    for (String aggregateValue : aggregateValues) {
      // -----------------------------------------------------------------
      // Get params
      // -----------------------------------------------------------------

      String[] info = aggregateValue.split(SEPERATE_SIGN);

      int dataElementId = Integer.parseInt(info[0]);
      int optionComboId = Integer.parseInt(info[1]);
      String periodIsoId = info[2];
      int orgunitId = Integer.parseInt(info[3]);
      String resultValue = info[4];

      // -----------------------------------------------------------------
      // Create objects
      // -----------------------------------------------------------------

      DataElement dataElement = dataElementService.getDataElement(dataElementId);
      DataElementCategoryOptionCombo optionCombo =
          categoryService.getDataElementCategoryOptionCombo(optionComboId);

      DataElementCategoryOptionCombo attributeOptioncombo =
          categoryService.getDefaultDataElementCategoryOptionCombo();

      Period period = PeriodType.getPeriodFromIsoString(periodIsoId);

      OrganisationUnit orgunit = organisationUnitService.getOrganisationUnit(orgunitId);

      DataValue dataValue =
          dataValueService.getDataValue(
              dataElement, period, orgunit, optionCombo, attributeOptioncombo);

      // -----------------------------------------------------------------
      // Save/Update/Delete data-values
      // -----------------------------------------------------------------

      if (resultValue != "0.0") {
        if (dataValue == null) {
          dataValue =
              new DataValue(
                  dataElement,
                  period,
                  orgunit,
                  optionCombo,
                  attributeOptioncombo,
                  "" + resultValue,
                  CaseAggregationCondition.AUTO_STORED_BY,
                  new Date(),
                  null);
          dataValueService.addDataValue(dataValue);
        } else {
          dataValue.setValue(resultValue);
          dataValue.setLastUpdated(new Date());
          dataValue.setStoredBy(CaseAggregationCondition.AUTO_STORED_BY);

          dataValueService.updateDataValue(dataValue);
        }
      } else if (dataValue != null) {
        dataValueService.deleteDataValue(dataValue);
      }
    }

    return SUCCESS;
  }
  @Transactional
  public void export(
      Collection<Integer> dataElementIds,
      Collection<Integer> indicatorIds,
      Collection<Integer> periodIds,
      Collection<Integer> organisationUnitIds,
      Collection<Integer> organisationUnitGroupIds,
      TaskId id) {
    final int cpuCores = SystemUtils.getCpuCores();

    Clock clock =
        new Clock()
            .startClock()
            .logTime(
                "Data mart export process started, number of CPU cores: "
                    + cpuCores
                    + ", "
                    + SystemUtils.getMemoryString());
    notifier.clear(id, DATAMART).notify(id, DATAMART, "Data mart export process started");

    // ---------------------------------------------------------------------
    // Recreate temporary tables
    // ---------------------------------------------------------------------

    dataMartManager.dropTempAggregatedTables();
    dataMartManager.createTempAggregatedTables();

    clock.logTime("Recreated temporary tables");

    // ---------------------------------------------------------------------
    // Replace null with empty collection
    // ---------------------------------------------------------------------

    dataElementIds = dataElementIds != null ? dataElementIds : new ArrayList<Integer>();
    indicatorIds = indicatorIds != null ? indicatorIds : new ArrayList<Integer>();
    periodIds = periodIds != null ? periodIds : new ArrayList<Integer>();
    organisationUnitIds =
        organisationUnitIds != null ? organisationUnitIds : new ArrayList<Integer>();
    organisationUnitGroupIds =
        organisationUnitGroupIds != null ? organisationUnitGroupIds : new ArrayList<Integer>();

    clock.logTime(
        "Data elements: "
            + dataElementIds.size()
            + ", indicators: "
            + indicatorIds.size()
            + ", periods: "
            + periodIds.size()
            + ", org units: "
            + organisationUnitIds.size());

    // ---------------------------------------------------------------------
    // Get objects
    // ---------------------------------------------------------------------

    final Collection<Indicator> indicators = indicatorService.getIndicators(indicatorIds);
    final Collection<Period> periods = periodService.getPeriods(periodIds);
    final List<OrganisationUnit> organisationUnits =
        new ArrayList<OrganisationUnit>(
            organisationUnitService.getOrganisationUnits(organisationUnitIds));
    final Collection<OrganisationUnitGroup> organisationUnitGroups =
        organisationUnitGroupService.getOrganisationUnitGroups(organisationUnitGroupIds);
    final Collection<DataElement> dataElements = dataElementService.getDataElements(dataElementIds);

    clock.logTime("Retrieved meta-data objects");
    notifier.notify(id, DATAMART, "Filtering meta-data");

    // ---------------------------------------------------------------------
    // Filter objects
    // ---------------------------------------------------------------------

    organisationUnitService.filterOrganisationUnitsWithoutData(organisationUnits);
    FilterUtils.filter(dataElements, new AggregatableDataElementFilter());
    FilterUtils.filter(dataElements, new DataElementWithAggregationFilter());
    expressionService.filterInvalidIndicators(indicators);

    clock.logTime("Filtered objects");
    notifier.notify(id, DATAMART, "Loading indicators");

    // ---------------------------------------------------------------------
    // Explode indicator expressions
    // ---------------------------------------------------------------------

    for (Indicator indicator : indicators) {
      indicator.setExplodedNumerator(expressionService.explodeExpression(indicator.getNumerator()));
      indicator.setExplodedDenominator(
          expressionService.explodeExpression(indicator.getDenominator()));
    }

    clock.logTime("Exploded indicator expressions");
    notifier.notify(id, DATAMART, "Loading data elements");

    // ---------------------------------------------------------------------
    // Get operands
    // ---------------------------------------------------------------------

    final Collection<DataElementOperand> dataElementOperands =
        categoryService.getOperands(dataElements);
    final List<DataElementOperand> indicatorOperands =
        new ArrayList<DataElementOperand>(
            categoryService.populateOperands(
                expressionService.getOperandsInIndicators(indicators)));

    Set<DataElementOperand> allOperands = new HashSet<DataElementOperand>();
    allOperands.addAll(dataElementOperands);
    allOperands.addAll(indicatorOperands);

    clock.logTime("Retrieved operands: " + allOperands.size());
    notifier.notify(id, DATAMART, "Loading periods");

    // ---------------------------------------------------------------------
    // Filter out future periods
    // ---------------------------------------------------------------------

    FilterUtils.filter(periods, new PastAndCurrentPeriodFilter());

    clock.logTime("Number of periods: " + periods.size());
    notifier.notify(id, DATAMART, "Filtering data elements without data");

    // ---------------------------------------------------------------------
    // Remove operands without data
    // ---------------------------------------------------------------------

    allOperands = dataMartManager.getOperandsWithData(allOperands);

    indicatorOperands.retainAll(allOperands);

    clock.logTime(
        "Number of operands with data: "
            + allOperands.size()
            + ", "
            + SystemUtils.getMemoryString());
    notifier.notify(id, DATAMART, "Populating crosstabulation table");

    // ---------------------------------------------------------------------
    // Create crosstabtable
    // ---------------------------------------------------------------------

    final Collection<Integer> intersectingPeriodIds =
        ConversionUtils.getIdentifiers(Period.class, periodService.getIntersectionPeriods(periods));
    final PeriodHierarchy periodHierarchy = periodService.getPeriodHierarchy(periods);
    final Set<Integer> orgUnitChildrenIds =
        organisationUnitService.getOrganisationUnitHierarchy().getChildren(organisationUnitIds);
    final List<Integer> crossTabOrgUnitIds = new ArrayList<Integer>(orgUnitChildrenIds);

    final String key = crossTabService.createCrossTabTable(crossTabOrgUnitIds);

    final List<DataElementOperand> operandList = new ArrayList<DataElementOperand>(allOperands);
    Collections.shuffle(operandList);

    final List<List<DataElementOperand>> operandPages =
        new PaginatedList<DataElementOperand>(operandList).setNumberOfPages(cpuCores).getPages();

    List<Future<?>> crossTabFutures = new ArrayList<Future<?>>();

    for (List<DataElementOperand> operandPage : operandPages) {
      crossTabFutures.add(
          crossTabService.populateCrossTabTable(
              operandPage, intersectingPeriodIds, crossTabOrgUnitIds, key));
    }

    ConcurrentUtils.waitForCompletion(crossTabFutures);

    clock.logTime("Populated crosstab table, " + SystemUtils.getMemoryString());
    notifier.notify(id, DATAMART, "Exporting data element data");

    final boolean isDataElements = true;

    final boolean isIndicators = indicators != null && indicators.size() > 0;

    final int groupLevel =
        (Integer)
            systemSettingManager.getSystemSetting(
                KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL);

    final boolean isGroups =
        organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0;

    if (isDataElements) {
      // -----------------------------------------------------------------
      // 1. Export data element values
      // -----------------------------------------------------------------

      if (operandList.size() > 0) {
        final OrganisationUnitHierarchy orgUnitHierarchy =
            organisationUnitService
                .getOrganisationUnitHierarchy()
                .prepareChildren(organisationUnits);

        List<Future<?>> futures = new ArrayList<Future<?>>();

        for (List<DataElementOperand> operandPage : operandPages) {
          futures.add(
              dataElementDataMart.exportDataValues(
                  operandPage,
                  periods,
                  organisationUnits,
                  null,
                  periodHierarchy,
                  orgUnitHierarchy,
                  AggregatedDataValueTempBatchHandler.class,
                  key));
        }

        ConcurrentUtils.waitForCompletion(futures);
      }

      clock.logTime(
          "Exported values for data element operands ("
              + operandList.size()
              + "), pages: "
              + operandPages.size()
              + ", "
              + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Dropping data element index");

      // -----------------------------------------------------------------
      // 2. Drop data element index
      // -----------------------------------------------------------------

      dataMartManager.dropDataValueIndex();

      clock.logTime("Dropped data element index");
      notifier.notify(id, DATAMART, "Deleting existing data element data");

      // -----------------------------------------------------------------
      // 3. Delete existing aggregated data values
      // -----------------------------------------------------------------

      dataMartManager.deleteAggregatedDataValues(periodIds);

      clock.logTime("Deleted existing data element data");
      notifier.notify(id, DATAMART, "Copying data element data from temporary table");

      // -----------------------------------------------------------------
      // 4. Copy aggregated data values from temporary table
      // -----------------------------------------------------------------

      dataMartManager.copyAggregatedDataValuesFromTemp();

      clock.logTime("Copied data element data from temporary table");
      notifier.notify(id, DATAMART, "Creating data element index");

      // -----------------------------------------------------------------
      // 5. Create data element index
      // -----------------------------------------------------------------

      dataMartManager.createDataValueIndex();

      clock.logTime("Created data element index");
    }

    if (isGroups && isDataElements) {
      // -----------------------------------------------------------------
      // 1. Export data element values
      // -----------------------------------------------------------------

      notifier.notify(id, DATAMART, "Exporting data element org unit data");

      Collection<OrganisationUnit> groupOrganisationUnits =
          new HashSet<OrganisationUnit>(organisationUnits);

      FilterUtils.filter(
          groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter(groupLevel));

      if (operandList.size() > 0) {
        final OrganisationUnitHierarchy orgUnitHierarchy =
            organisationUnitService
                .getOrganisationUnitHierarchy()
                .prepareChildren(groupOrganisationUnits, organisationUnitGroups);

        List<Future<?>> futures = new ArrayList<Future<?>>();

        for (List<DataElementOperand> operandPage : operandPages) {
          futures.add(
              dataElementDataMart.exportDataValues(
                  operandPage,
                  periods,
                  groupOrganisationUnits,
                  organisationUnitGroups,
                  periodHierarchy,
                  orgUnitHierarchy,
                  AggregatedOrgUnitDataValueTempBatchHandler.class,
                  key));
        }

        ConcurrentUtils.waitForCompletion(futures);
      }

      clock.logTime(
          "Exported values for data element operands ("
              + operandList.size()
              + "), pages: "
              + operandPages.size()
              + ", "
              + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Dropping data element data indexes");

      // -----------------------------------------------------------------
      // 2. Drop data element index
      // -----------------------------------------------------------------

      dataMartManager.dropOrgUnitDataValueIndex();

      clock.logTime("Dropped org unit data element index");
      notifier.notify(id, DATAMART, "Deleting existing org unit data element data");

      // -----------------------------------------------------------------
      // 3. Delete existing aggregated data values
      // -----------------------------------------------------------------

      dataMartManager.deleteAggregatedOrgUnitDataValues(periodIds);

      clock.logTime("Deleted existing aggregated org unit datavalues");
      notifier.notify(id, DATAMART, "Copying org unit data element data");

      // -----------------------------------------------------------------
      // 4. Copy aggregated org unit data values from temporary table
      // -----------------------------------------------------------------

      dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp();

      clock.logTime("Copied org unit data element data from temporary table");
      notifier.notify(id, DATAMART, "Creating org unit data element index");

      // -----------------------------------------------------------------
      // 5. Create org unit data element index
      // -----------------------------------------------------------------

      dataMartManager.createOrgUnitDataValueIndex();

      clock.logTime("Created org unit data element index");
      notifier.notify(id, DATAMART, "Exporting data for org unit indicator data");
    }

    crossTabService.dropCrossTabTable(key);

    List<List<OrganisationUnit>> organisationUnitPages =
        new PaginatedList<OrganisationUnit>(organisationUnits)
            .setNumberOfPages(cpuCores)
            .getPages();

    if (isIndicators) {
      // -----------------------------------------------------------------
      // 1. Create and populate aggregated data cache
      // -----------------------------------------------------------------

      notifier.notify(id, DATAMART, "Populating aggregated data cache");

      crossTabService.createAggregatedDataCache(indicatorOperands, key);

      List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();

      for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
        aggregatedDataCacheFutures.add(
            crossTabService.populateAggregatedDataCache(
                indicatorOperands, periods, organisationUnitPage, key));
      }

      ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);

      clock.logTime(
          "Created aggregated data cache, number of indicator operands: "
              + indicatorOperands.size());
      notifier.notify(id, DATAMART, "Exporting indicator data");

      // -----------------------------------------------------------------
      // 2. Export indicator values
      // -----------------------------------------------------------------

      List<Future<?>> futures = new ArrayList<Future<?>>();

      for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
        futures.add(
            indicatorDataMart.exportIndicatorValues(
                indicators,
                periods,
                organisationUnitPage,
                null,
                indicatorOperands,
                AggregatedIndicatorValueTempBatchHandler.class,
                key));
      }

      ConcurrentUtils.waitForCompletion(futures);

      clock.logTime(
          "Exported values for indicators ("
              + indicators.size()
              + "), pages: "
              + organisationUnitPages.size()
              + ", "
              + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Dropping indicator index");

      // -----------------------------------------------------------------
      // 3. Drop aggregated data cache and indicator index
      // -----------------------------------------------------------------

      crossTabService.dropAggregatedDataCache(key);
      dataMartManager.dropIndicatorValueIndex();

      clock.logTime("Dropped indicator index, " + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Deleting existing indicator data");

      // -----------------------------------------------------------------
      // 4. Delete existing aggregated indicator values
      // -----------------------------------------------------------------

      dataMartManager.deleteAggregatedIndicatorValues(periodIds);

      clock.logTime("Deleted existing indicator data");
      notifier.notify(id, DATAMART, "Copying indicator data from temporary table");

      // -----------------------------------------------------------------
      // 5. Copy aggregated data values from temporary table
      // -----------------------------------------------------------------

      dataMartManager.copyAggregatedIndicatorValuesFromTemp();

      clock.logTime("Copied indicator data from temporary table");
      notifier.notify(id, DATAMART, "Creating indicator index");

      // -----------------------------------------------------------------
      // 6. Create indicator index
      // -----------------------------------------------------------------

      dataMartManager.createIndicatorValueIndex();

      clock.logTime("Created indicator index");
    }

    if (isGroups && isIndicators) {
      // -----------------------------------------------------------------
      // 1. Create aggregated data cache
      // -----------------------------------------------------------------

      notifier.notify(id, DATAMART, "Populating aggregated data cache");

      crossTabService.createAggregatedOrgUnitDataCache(indicatorOperands, key);

      List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();

      for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
        aggregatedDataCacheFutures.add(
            crossTabService.populateAggregatedOrgUnitDataCache(
                indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key));
      }

      ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);

      clock.logTime("Created aggregated org unit data cache");
      notifier.notify(id, DATAMART, "Exporting org unit indicator data");

      // -----------------------------------------------------------------
      // 2. Export indicator values
      // -----------------------------------------------------------------

      List<Future<?>> futures = new ArrayList<Future<?>>();

      for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
        futures.add(
            indicatorDataMart.exportIndicatorValues(
                indicators,
                periods,
                organisationUnitPage,
                organisationUnitGroups,
                indicatorOperands,
                AggregatedOrgUnitIndicatorValueTempBatchHandler.class,
                key));
      }

      ConcurrentUtils.waitForCompletion(futures);

      clock.logTime(
          "Exported values for indicators ("
              + indicators.size()
              + "), pages: "
              + organisationUnitPages.size()
              + ", "
              + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Dropping org unit indicator index");

      // -----------------------------------------------------------------
      // 3. Drop aggregated data cache and indicator index
      // -----------------------------------------------------------------

      crossTabService.dropAggregatedOrgUnitDataCache(key);
      dataMartManager.dropOrgUnitIndicatorValueIndex();

      clock.logTime("Dropped org unit indicator index, " + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Deleting existing org unit indicator data");

      // -----------------------------------------------------------------
      // 4. Delete existing aggregated indicator values
      // -----------------------------------------------------------------

      dataMartManager.deleteAggregatedOrgUnitIndicatorValues(periodIds);

      clock.logTime("Deleted existing aggregated org unit indicatorvalues");
      notifier.notify(id, DATAMART, "Copying org unit indicator data from temporary table");

      // -----------------------------------------------------------------
      // 5. Copy aggregated org unit indicator values from temp table
      // -----------------------------------------------------------------

      dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();

      clock.logTime("Copied org unit indicator data from temporary table");
      notifier.notify(id, DATAMART, "Creating org unit indicator indexes");

      // -----------------------------------------------------------------
      // 6. Create org unit indicator index
      // -----------------------------------------------------------------

      dataMartManager.createOrgUnitIndicatorValueIndex();

      clock.logTime("Created org unit indicator index");
    }

    // ---------------------------------------------------------------------
    // Drop temporary tables
    // ---------------------------------------------------------------------

    dataMartManager.dropTempAggregatedTables();

    clock.logTime("Dropped crosstab table");
    clock.logTime("Data mart export process completed");
    notifier.notify(id, DATAMART, INFO, "Data mart process completed", true);
  }
  @Override
  public void setUpTest() {
    mockDataValueBatchHandler = new MockBatchHandler<>();
    mockBatchHandlerFactory = new MockBatchHandlerFactory();
    mockBatchHandlerFactory.registerBatchHandler(
        DataValueBatchHandler.class, mockDataValueBatchHandler);
    setDependency(dataValueSetService, "batchHandlerFactory", mockBatchHandlerFactory);

    categoryOptionA = createCategoryOption('A');
    categoryOptionB = createCategoryOption('B');
    categoryA = createDataElementCategory('A', categoryOptionA, categoryOptionB);
    categoryComboA = createCategoryCombo('A', categoryA);
    ocDef = categoryService.getDefaultDataElementCategoryOptionCombo();

    ocA = createCategoryOptionCombo(categoryComboA, categoryOptionA);
    ocB = createCategoryOptionCombo(categoryComboA, categoryOptionB);
    deA = createDataElement('A');
    deB = createDataElement('B');
    deC = createDataElement('C');
    deD = createDataElement('D');
    dsA = createDataSet('A', new MonthlyPeriodType());
    ouA = createOrganisationUnit('A');
    ouB = createOrganisationUnit('B');
    ouC = createOrganisationUnit('C');
    peA =
        createPeriod(
            PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME),
            getDate(2012, 1, 1),
            getDate(2012, 1, 31));
    peB =
        createPeriod(
            PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME),
            getDate(2012, 2, 1),
            getDate(2012, 2, 29));

    ocA.setUid("kjuiHgy67hg");
    ocB.setUid("Gad33qy67g5");
    deA.setUid("f7n9E0hX8qk");
    deB.setUid("Ix2HsbDMLea");
    deC.setUid("eY5ehpbEsB7");
    dsA.setUid("pBOMPrpg1QX");
    ouA.setUid("DiszpKrYNg8");
    ouB.setUid("BdfsJfj87js");
    ouC.setUid("j7Hg26FpoIa");

    ocA.setCode("OC_A");
    ocB.setCode("OC_B");
    deA.setCode("DE_A");
    deB.setCode("DE_B");
    deC.setCode("DE_C");
    deD.setCode("DE_D");
    dsA.setCode("DS_A");
    ouA.setCode("OU_A");
    ouB.setCode("OU_B");
    ouC.setCode("OU_C");

    categoryService.addDataElementCategoryOption(categoryOptionA);
    categoryService.addDataElementCategoryOption(categoryOptionB);
    categoryService.addDataElementCategory(categoryA);
    categoryService.addDataElementCategoryCombo(categoryComboA);
    categoryService.addDataElementCategoryOptionCombo(ocA);
    categoryService.addDataElementCategoryOptionCombo(ocB);

    dataElementService.addDataElement(deA);
    dataElementService.addDataElement(deB);
    dataElementService.addDataElement(deC);
    dataElementService.addDataElement(deD);
    dataSetService.addDataSet(dsA);
    organisationUnitService.addOrganisationUnit(ouA);
    organisationUnitService.addOrganisationUnit(ouB);
    organisationUnitService.addOrganisationUnit(ouC);
    periodService.addPeriod(peA);
    periodService.addPeriod(peB);

    user = createUser('A');
    user.setOrganisationUnits(Sets.newHashSet(ouA, ouB));
    CurrentUserService currentUserService = new MockCurrentUserService(user);
    setDependency(dataValueSetService, "currentUserService", currentUserService);
  }