@Test
  public void testAverageIntDataElement() {
    DataElement dataElement = createDataElement('A', categoryCombo);

    dataElement.setType(DataElement.VALUE_TYPE_INT);
    dataElement.setAggregationOperator(DataElement.AGGREGATION_OPERATOR_AVERAGE_SUM);

    dataElementIds.add(dataElementService.addDataElement(dataElement));

    DataSet dataSet = createDataSet('A', new YearlyPeriodType());
    dataSet.getDataElements().add(dataElement);
    dataSetService.addDataSet(dataSet);
    dataElement.getDataSets().add(dataSet);
    dataElementService.updateDataElement(dataElement);

    Period dataPeriod =
        createPeriod(new YearlyPeriodType(), getDate(2000, 1, 1), getDate(2000, 12, 31));

    OrganisationUnit unit = createOrganisationUnit('A');

    organisationUnitIds.add(organisationUnitService.addOrganisationUnit(unit));

    dataValueService.addDataValue(
        createDataValue(dataElement, dataPeriod, unit, "100", categoryOptionCombo));

    Period periodA =
        createPeriod(new MonthlyPeriodType(), getDate(2000, 3, 1), getDate(2000, 3, 31));
    Period periodB =
        createPeriod(new MonthlyPeriodType(), getDate(2000, 4, 1), getDate(2000, 4, 30));
    Period periodC =
        createPeriod(new QuarterlyPeriodType(), getDate(2000, 3, 1), getDate(2000, 5, 31));

    periodIds.add(periodService.addPeriod(periodA));
    periodIds.add(periodService.addPeriod(periodB));
    periodIds.add(periodService.addPeriod(periodC));

    dataMartEngine.export(dataElementIds, indicatorIds, periodIds, organisationUnitIds);

    assertEquals(
        100.0,
        aggregatedDataValueService.getAggregatedValue(
            dataElement, categoryOptionCombo, periodA, unit),
        DELTA);
    assertEquals(
        100.0,
        aggregatedDataValueService.getAggregatedValue(
            dataElement, categoryOptionCombo, periodB, unit),
        DELTA);
    assertEquals(
        100.0,
        aggregatedDataValueService.getAggregatedValue(
            dataElement, categoryOptionCombo, periodC, unit),
        DELTA);
  }
  public String execute() throws Exception {
    // ---------------------------------------------------------------------
    // Prepare values
    // ---------------------------------------------------------------------

    code = nullIfEmpty(code);
    shortName = nullIfEmpty(shortName);
    description = nullIfEmpty(description);

    PeriodType periodType = PeriodType.getPeriodTypeByName(frequencySelect);

    DataSet dataSet = new DataSet(name, shortName, code, periodType);

    MapLegendSet legendSet = mappingService.getMapLegendSet(selectedLegendSetId);

    dataSet.setExpiryDays(expiryDays);
    dataSet.setTimelyDays(timelyDays);
    dataSet.setSkipAggregation(skipAggregation);

    for (String id : dataElementsSelectedList) {
      dataSet.addDataElement(dataElementService.getDataElement(Integer.parseInt(id)));
    }

    Set<Indicator> indicators = new HashSet<Indicator>();

    for (String id : indicatorsSelectedList) {
      indicators.add(indicatorService.getIndicator(Integer.parseInt(id)));
    }

    if (categoryComboId != null) {
      dataSet.setCategoryCombo(categoryService.getDataElementCategoryCombo(categoryComboId));
    }

    dataSet.setDescription(description);
    dataSet.setVersion(1);
    dataSet.setMobile(false);
    dataSet.setIndicators(indicators);
    dataSet.setNotificationRecipients(userGroupService.getUserGroup(notificationRecipients));
    dataSet.setAllowFuturePeriods(allowFuturePeriods);
    dataSet.setFieldCombinationRequired(fieldCombinationRequired);
    dataSet.setValidCompleteOnly(validCompleteOnly);
    dataSet.setNotifyCompletingUser(notifyCompletingUser);
    dataSet.setApproveData(approveData);
    dataSet.setSkipOffline(skipOffline);
    dataSet.setDataElementDecoration(dataElementDecoration);
    dataSet.setRenderAsTabs(renderAsTabs);
    dataSet.setRenderHorizontally(renderHorizontally);
    dataSet.setLegendSet(legendSet);

    dataSetService.addDataSet(dataSet);

    userService.assignDataSetToUserRole(dataSet);

    return SUCCESS;
  }
  @Override
  public String execute() {
    if (dataElementGroupId > 0) {
      DataElementGroup group =
          dataElementService.getDataElementGroup(dataElementGroupId.intValue());

      dataElements = new ArrayList<>(group.getMembers());
    } else {
      dataElements = new ArrayList<>(dataElementService.getAllDataElements());
    }

    Collections.sort(dataElements, new IdentifiableObjectNameComparator());

    for (DataElement dataElement : dataElements) {
      DataElementCategoryCombo categoryCombo = dataElement.getCategoryCombo();

      Set<DataElementCategoryOptionCombo> optionCombos = categoryCombo.getOptionCombos();

      if (optionCombos.size() > 1) {
        for (DataElementCategoryOptionCombo optionCombo : optionCombos) {
          DataElementOperand operand =
              new DataElementOperand(
                  dataElement.getUid(),
                  optionCombo.getUid(),
                  dataElement.getName() + optionCombo.getName());
          operands.add(operand);
        }
      } else {
        DataElementOperand operand =
            new DataElementOperand(
                dataElement.getUid(),
                optionCombos.iterator().next().getUid(),
                dataElement.getName());
        operands.add(operand);
      }
    }

    return SUCCESS;
  }
Exemplo n.º 4
0
  @Override
  public void setUpTest() {
    organisationUnit = createOrganisationUnit('A');
    organisationUnitLevel = new OrganisationUnitLevel(1, "Level");

    organisationUnitService.addOrganisationUnit(organisationUnit);
    organisationUnitService.addOrganisationUnitLevel(organisationUnitLevel);

    indicatorGroup = createIndicatorGroup('A');
    indicatorService.addIndicatorGroup(indicatorGroup);

    indicatorType = createIndicatorType('A');
    indicatorService.addIndicatorType(indicatorType);

    indicator = createIndicator('A', indicatorType);
    indicatorService.addIndicator(indicator);

    dataElement = createDataElement('A');
    dataElementService.addDataElement(dataElement);

    dataElementGroup = createDataElementGroup('A');
    dataElementGroup.getMembers().add(dataElement);
    dataElementService.addDataElementGroup(dataElementGroup);

    periodType = periodService.getPeriodTypeByName(MonthlyPeriodType.NAME);
    period = createPeriod(periodType, getDate(2000, 1, 1), getDate(2000, 2, 1));
    periodService.addPeriod(period);

    mapLegendSet = createLegendSet('A');
    legendService.addLegendSet(mapLegendSet);

    internalMapLayer = new InternalMapLayer();
    internalMapLayer.setRadiusLow(15);
    internalMapLayer.setRadiusHigh(35);
    internalMapLayer.setColorLow(Color.YELLOW);
    internalMapLayer.setColorHigh(Color.RED);
    internalMapLayer.setOpacity(0.5f);
  }
  @Override
  public void setUpTest() {
    organisationUnit = createOrganisationUnit('A');
    organisationUnitService.addOrganisationUnit(organisationUnit);

    Program program = createProgram('A', new HashSet<ProgramStage>(), organisationUnit);
    programService.addProgram(program);

    stageA = new ProgramStage("A", program);
    programStageService.saveProgramStage(stageA);

    DataElement dataElementA = createDataElement('A');
    DataElement dataElementB = createDataElement('B');

    dataElementService.addDataElement(dataElementA);
    dataElementService.addDataElement(dataElementB);

    ProgramStageDataElement stageDeA = new ProgramStageDataElement(stageA, dataElementA, false, 1);
    ProgramStageDataElement stageDeB = new ProgramStageDataElement(stageA, dataElementB, false, 2);

    programStageDataElementService.addProgramStageDataElement(stageDeA);
    programStageDataElementService.addProgramStageDataElement(stageDeB);

    List<ProgramStageDataElement> psDataElements = new ArrayList<>();
    psDataElements.add(stageDeA);
    psDataElements.add(stageDeB);

    stageB = new ProgramStage("B", program);
    programStageService.saveProgramStage(stageB);

    Set<ProgramStage> programStages = new HashSet<>();
    programStages.add(stageA);
    programStages.add(stageB);
    program.setProgramStages(programStages);
    programService.updateProgram(program);

    sectionA = new ProgramStageSection();
    sectionA.setName("A");
    sectionA.setProgramStageDataElements(psDataElements);
    sectionA.setSortOrder(1);

    sectionB = new ProgramStageSection();
    sectionB.setName("B");
    sectionB.setSortOrder(2);

    Set<ProgramStageSection> sections = new HashSet<>();
    sections.add(sectionA);
    sections.add(sectionB);
    stageA.setProgramStageSections(sections);
  }
  public String execute() {
    // ---------------------------------------------------------------------
    // Get group members
    // ---------------------------------------------------------------------

    if (id != null) {
      DataElementGroup group = dataElementService.getDataElementGroup(id);

      groupMembers = new ArrayList<DataElement>(group.getMembers());

      Collections.sort(groupMembers, new IdentifiableObjectNameComparator());
    }

    return SUCCESS;
  }
  protected void addDataValue(
      OrganisationUnit unit,
      Period period,
      String expression,
      String value,
      Set<DataValue> oldList,
      Set<DataValue> newList) {
    // value = value.replaceAll( "\\.", "" ).replace( ",", "." );

    DataElementOperand operand =
        expressionService.getOperandsInExpression(expression).iterator().next();

    DataElement dataElement = dataElementService.getDataElement(operand.getDataElementId());

    DataElementCategoryOptionCombo optionCombo =
        categoryService.getDataElementCategoryOptionCombo(operand.getOptionComboId());

    String storedBy = currentUserService.getCurrentUsername();

    DataValue dataValue = dataValueService.getDataValue(unit, dataElement, period, optionCombo);

    if (dataValue == null) {
      dataValue =
          new DataValue(dataElement, period, unit, value, storedBy, new Date(), null, optionCombo);
      dataValueService.addDataValue(dataValue);

      newList.add(dataValue);
    } else {
      DataValue backedUpDataValue =
          new DataValue(dataElement, period, unit, dataValue.getValue(), optionCombo);

      oldList.add(backedUpDataValue);

      dataValue.setValue(value);
      dataValue.setTimestamp(new Date());
      dataValue.setStoredBy(storedBy);

      dataValueService.updateDataValue(dataValue);
    }
  }
Exemplo n.º 8
0
  public String execute() {
    DataElement dataElement = dataElementService.getDataElement(dataElementId);
    DataElementCategoryOptionCombo categoryOptionCombo =
        categoryService.getDataElementCategoryOptionCombo(categoryOptionComboId);
    Period period = PeriodType.createPeriodExternalId(periodId);
    OrganisationUnit source = organisationUnitService.getOrganisationUnit(organisationUnitId);

    DataValue dataValue =
        dataValueService.getDataValue(source, dataElement, period, categoryOptionCombo);

    boolean isMarked = dataValue.isFollowup();

    dataValue.setFollowup(!isMarked);

    dataValueService.updateDataValue(dataValue);

    message = !isMarked ? "marked" : "unmarked";

    log.info(!isMarked ? "Data value marked for follow-up" : "Data value unmarked for follow-up");

    return SUCCESS;
  }
  private void applyAggregationLevels(List<AnalyticsTable> tables) {
    int maxLevels = organisationUnitService.getNumberOfOrganisationalLevels();

    boolean hasAggLevels = false;

    levelLoop:
    for (int i = 0; i < maxLevels; i++) {
      int level = maxLevels - i;

      Collection<String> dataElements =
          IdentifiableObjectUtils.getUids(
              dataElementService.getDataElementsByAggregationLevel(level));

      if (dataElements.isEmpty()) {
        continue levelLoop;
      }

      hasAggLevels = true;

      ConcurrentLinkedQueue<AnalyticsTable> tableQ = new ConcurrentLinkedQueue<>(tables);

      List<Future<?>> futures = new ArrayList<>();

      for (int j = 0; j < getProcessNo(); j++) {
        futures.add(tableManager.applyAggregationLevels(tableQ, dataElements, level));
      }

      ConcurrentUtils.waitForCompletion(futures);
    }

    if (hasAggLevels) {
      vacuumTables(tables);

      log.info("Vacuumed tables");
    }
  }
Exemplo n.º 10
0
  @Override
  public String execute() throws Exception {
    ExportParams params = new ExportParams();

    if (dataValue || dataValueDaily) {
      params.setCategories(null);
      params.setCategoryCombos(null);
      params.setCategoryOptions(null);
      params.setCategoryOptionCombos(null);
      /*
       * params.setDataElementGroups( null );
       * params.setDataElementGroupSets( null ); params.setIndicators(
       * null ); params.setIndicatorTypes( null );
       * params.setIndicatorGroups( null ); params.setIndicatorGroupSets(
       * null ); params.setDataDictionaries( null ); params.setDataSets(
       * null ); params.setOrganisationUnitGroups( null );
       * params.setOrganisationUnitGroupSets( null );
       * params.setOrganisationUnitLevels( null );
       * params.setValidationRules( null ); params.setReports( null );
       * params.setReportTables( null ); params.setPeriods( null ); //
       * TODO Include only relevant periods params.setCharts( null );
       * params.setPeriods( null );
       */

      Set<Integer> dataElementz = new HashSet<>();

      Collection<DataElement> children = dataElementService.getAllDataElements();

      for (DataElement child : children) {
        dataElementz.add(child.getId());
      }

      params.setDataElements(dataElementz);

      Set<Integer> orgUnits = new HashSet<>();

      Collection<OrganisationUnit> orgChildren = organisationUnitService.getAllOrganisationUnits();

      for (OrganisationUnit child : orgChildren) {
        orgUnits.add(child.getId());
      }

      params.setOrganisationUnits(orgUnits);
    }

    if (dataValue) {
      params.setMetaData(false);
    }

    if (dataValueDaily) {
      params.setMetaData(false);
    }

    // params.setMetaData( true );
    params.setIncludeDataValues(true);
    params.setCurrentUser(currentUserService.getCurrentUser());
    params.setDataValue(dataValue);
    params.setDataValueDaily(dataValueDaily);

    params.setI18n(i18n);
    params.setFormat(format);
    params.setStartDate(getMediumDate(startDate));
    params.setEndDate(getMediumDate(endDate));

    ExportService exportService = serviceProvider.provide(exportFormat);

    inputStream = exportService.exportData(params);

    fileName = FILENAME;

    return SUCCESS;
  }
Exemplo n.º 11
0
  @RequestMapping(method = RequestMethod.POST, produces = "text/plain")
  public void saveDataValue(
      @RequestParam String de,
      @RequestParam String cc,
      @RequestParam String pe,
      @RequestParam String ou,
      @RequestParam String value,
      HttpServletResponse response) {
    DataElement dataElement = dataElementService.getDataElement(de);

    if (dataElement == null) {
      ContextUtils.conflictResponse(response, "Illegal data element identifier: " + de);
      return;
    }

    DataElementCategoryOptionCombo categoryOptionCombo =
        categoryService.getDataElementCategoryOptionCombo(cc);

    if (categoryOptionCombo == null) {
      ContextUtils.conflictResponse(response, "Illegal category option combo identifier: " + cc);
      return;
    }

    Period period = PeriodType.getPeriodFromIsoString(pe);

    if (period == null) {
      ContextUtils.conflictResponse(response, "Illegal period identifier: " + pe);
      return;
    }

    OrganisationUnit organisationUnit = organisationUnitService.getOrganisationUnit(ou);

    if (organisationUnit == null) {
      ContextUtils.conflictResponse(response, "Illegal organisation unit identifier: " + ou);
      return;
    }

    if (dataSetService.isLocked(dataElement, period, organisationUnit, null)) {
      ContextUtils.conflictResponse(response, "Data set is locked");
      return;
    }

    value = StringUtils.trimToNull(value);

    String storedBy = currentUserService.getCurrentUsername();

    Date now = new Date();

    DataValue dataValue =
        dataValueService.getDataValue(organisationUnit, dataElement, period, categoryOptionCombo);

    if (dataValue == null) {
      if (value != null) {
        dataValue =
            new DataValue(
                dataElement,
                period,
                organisationUnit,
                value,
                storedBy,
                now,
                null,
                categoryOptionCombo);
        dataValueService.addDataValue(dataValue);
      }
    } else {
      dataValue.setValue(value);
      dataValue.setTimestamp(now);
      dataValue.setStoredBy(storedBy);

      dataValueService.updateDataValue(dataValue);
    }
  }
Exemplo n.º 12
0
  @Override
  public void setUpTest() {
    mockDataValueBatchHandler = new MockBatchHandler<>();
    mockBatchHandlerFactory = new MockBatchHandlerFactory();
    mockBatchHandlerFactory.registerBatchHandler(
        DataValueBatchHandler.class, mockDataValueBatchHandler);
    setDependency(dataValueSetService, "batchHandlerFactory", mockBatchHandlerFactory);

    categoryOptionA = createCategoryOption('A');
    categoryOptionB = createCategoryOption('B');
    categoryA = createDataElementCategory('A', categoryOptionA, categoryOptionB);
    categoryComboA = createCategoryCombo('A', categoryA);
    ocDef = categoryService.getDefaultDataElementCategoryOptionCombo();

    ocA = createCategoryOptionCombo(categoryComboA, categoryOptionA);
    ocB = createCategoryOptionCombo(categoryComboA, categoryOptionB);
    deA = createDataElement('A');
    deB = createDataElement('B');
    deC = createDataElement('C');
    deD = createDataElement('D');
    dsA = createDataSet('A', new MonthlyPeriodType());
    ouA = createOrganisationUnit('A');
    ouB = createOrganisationUnit('B');
    ouC = createOrganisationUnit('C');
    peA =
        createPeriod(
            PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME),
            getDate(2012, 1, 1),
            getDate(2012, 1, 31));
    peB =
        createPeriod(
            PeriodType.getByNameIgnoreCase(MonthlyPeriodType.NAME),
            getDate(2012, 2, 1),
            getDate(2012, 2, 29));

    ocA.setUid("kjuiHgy67hg");
    ocB.setUid("Gad33qy67g5");
    deA.setUid("f7n9E0hX8qk");
    deB.setUid("Ix2HsbDMLea");
    deC.setUid("eY5ehpbEsB7");
    dsA.setUid("pBOMPrpg1QX");
    ouA.setUid("DiszpKrYNg8");
    ouB.setUid("BdfsJfj87js");
    ouC.setUid("j7Hg26FpoIa");

    ocA.setCode("OC_A");
    ocB.setCode("OC_B");
    deA.setCode("DE_A");
    deB.setCode("DE_B");
    deC.setCode("DE_C");
    deD.setCode("DE_D");
    dsA.setCode("DS_A");
    ouA.setCode("OU_A");
    ouB.setCode("OU_B");
    ouC.setCode("OU_C");

    categoryService.addDataElementCategoryOption(categoryOptionA);
    categoryService.addDataElementCategoryOption(categoryOptionB);
    categoryService.addDataElementCategory(categoryA);
    categoryService.addDataElementCategoryCombo(categoryComboA);
    categoryService.addDataElementCategoryOptionCombo(ocA);
    categoryService.addDataElementCategoryOptionCombo(ocB);

    dataElementService.addDataElement(deA);
    dataElementService.addDataElement(deB);
    dataElementService.addDataElement(deC);
    dataElementService.addDataElement(deD);
    dataSetService.addDataSet(dsA);
    organisationUnitService.addOrganisationUnit(ouA);
    organisationUnitService.addOrganisationUnit(ouB);
    organisationUnitService.addOrganisationUnit(ouC);
    periodService.addPeriod(peA);
    periodService.addPeriod(peB);

    user = createUser('A');
    user.setOrganisationUnits(Sets.newHashSet(ouA, ouB));
    CurrentUserService currentUserService = new MockCurrentUserService(user);
    setDependency(dataValueSetService, "currentUserService", currentUserService);
  }
Exemplo n.º 13
0
  @Override
  public String execute() throws Exception {
    for (String aggregateValue : aggregateValues) {
      // -----------------------------------------------------------------
      // Get params
      // -----------------------------------------------------------------

      String[] info = aggregateValue.split(SEPERATE_SIGN);

      int dataElementId = Integer.parseInt(info[0]);
      int optionComboId = Integer.parseInt(info[1]);
      String periodIsoId = info[2];
      int orgunitId = Integer.parseInt(info[3]);
      String resultValue = info[4];

      // -----------------------------------------------------------------
      // Create objects
      // -----------------------------------------------------------------

      DataElement dataElement = dataElementService.getDataElement(dataElementId);
      DataElementCategoryOptionCombo optionCombo =
          categoryService.getDataElementCategoryOptionCombo(optionComboId);

      DataElementCategoryOptionCombo attributeOptioncombo =
          categoryService.getDefaultDataElementCategoryOptionCombo();

      Period period = PeriodType.getPeriodFromIsoString(periodIsoId);

      OrganisationUnit orgunit = organisationUnitService.getOrganisationUnit(orgunitId);

      DataValue dataValue =
          dataValueService.getDataValue(
              dataElement, period, orgunit, optionCombo, attributeOptioncombo);

      // -----------------------------------------------------------------
      // Save/Update/Delete data-values
      // -----------------------------------------------------------------

      if (resultValue != "0.0") {
        if (dataValue == null) {
          dataValue =
              new DataValue(
                  dataElement,
                  period,
                  orgunit,
                  optionCombo,
                  attributeOptioncombo,
                  "" + resultValue,
                  CaseAggregationCondition.AUTO_STORED_BY,
                  new Date(),
                  null);
          dataValueService.addDataValue(dataValue);
        } else {
          dataValue.setValue(resultValue);
          dataValue.setLastUpdated(new Date());
          dataValue.setStoredBy(CaseAggregationCondition.AUTO_STORED_BY);

          dataValueService.updateDataValue(dataValue);
        }
      } else if (dataValue != null) {
        dataValueService.deleteDataValue(dataValue);
      }
    }

    return SUCCESS;
  }
Exemplo n.º 14
0
  @Transactional
  public void export(
      Collection<Integer> dataElementIds,
      Collection<Integer> indicatorIds,
      Collection<Integer> periodIds,
      Collection<Integer> organisationUnitIds,
      Collection<Integer> organisationUnitGroupIds,
      TaskId id) {
    final int cpuCores = SystemUtils.getCpuCores();

    Clock clock =
        new Clock()
            .startClock()
            .logTime(
                "Data mart export process started, number of CPU cores: "
                    + cpuCores
                    + ", "
                    + SystemUtils.getMemoryString());
    notifier.clear(id, DATAMART).notify(id, DATAMART, "Data mart export process started");

    // ---------------------------------------------------------------------
    // Recreate temporary tables
    // ---------------------------------------------------------------------

    dataMartManager.dropTempAggregatedTables();
    dataMartManager.createTempAggregatedTables();

    clock.logTime("Recreated temporary tables");

    // ---------------------------------------------------------------------
    // Replace null with empty collection
    // ---------------------------------------------------------------------

    dataElementIds = dataElementIds != null ? dataElementIds : new ArrayList<Integer>();
    indicatorIds = indicatorIds != null ? indicatorIds : new ArrayList<Integer>();
    periodIds = periodIds != null ? periodIds : new ArrayList<Integer>();
    organisationUnitIds =
        organisationUnitIds != null ? organisationUnitIds : new ArrayList<Integer>();
    organisationUnitGroupIds =
        organisationUnitGroupIds != null ? organisationUnitGroupIds : new ArrayList<Integer>();

    clock.logTime(
        "Data elements: "
            + dataElementIds.size()
            + ", indicators: "
            + indicatorIds.size()
            + ", periods: "
            + periodIds.size()
            + ", org units: "
            + organisationUnitIds.size());

    // ---------------------------------------------------------------------
    // Get objects
    // ---------------------------------------------------------------------

    final Collection<Indicator> indicators = indicatorService.getIndicators(indicatorIds);
    final Collection<Period> periods = periodService.getPeriods(periodIds);
    final List<OrganisationUnit> organisationUnits =
        new ArrayList<OrganisationUnit>(
            organisationUnitService.getOrganisationUnits(organisationUnitIds));
    final Collection<OrganisationUnitGroup> organisationUnitGroups =
        organisationUnitGroupService.getOrganisationUnitGroups(organisationUnitGroupIds);
    final Collection<DataElement> dataElements = dataElementService.getDataElements(dataElementIds);

    clock.logTime("Retrieved meta-data objects");
    notifier.notify(id, DATAMART, "Filtering meta-data");

    // ---------------------------------------------------------------------
    // Filter objects
    // ---------------------------------------------------------------------

    organisationUnitService.filterOrganisationUnitsWithoutData(organisationUnits);
    FilterUtils.filter(dataElements, new AggregatableDataElementFilter());
    FilterUtils.filter(dataElements, new DataElementWithAggregationFilter());
    expressionService.filterInvalidIndicators(indicators);

    clock.logTime("Filtered objects");
    notifier.notify(id, DATAMART, "Loading indicators");

    // ---------------------------------------------------------------------
    // Explode indicator expressions
    // ---------------------------------------------------------------------

    for (Indicator indicator : indicators) {
      indicator.setExplodedNumerator(expressionService.explodeExpression(indicator.getNumerator()));
      indicator.setExplodedDenominator(
          expressionService.explodeExpression(indicator.getDenominator()));
    }

    clock.logTime("Exploded indicator expressions");
    notifier.notify(id, DATAMART, "Loading data elements");

    // ---------------------------------------------------------------------
    // Get operands
    // ---------------------------------------------------------------------

    final Collection<DataElementOperand> dataElementOperands =
        categoryService.getOperands(dataElements);
    final List<DataElementOperand> indicatorOperands =
        new ArrayList<DataElementOperand>(
            categoryService.populateOperands(
                expressionService.getOperandsInIndicators(indicators)));

    Set<DataElementOperand> allOperands = new HashSet<DataElementOperand>();
    allOperands.addAll(dataElementOperands);
    allOperands.addAll(indicatorOperands);

    clock.logTime("Retrieved operands: " + allOperands.size());
    notifier.notify(id, DATAMART, "Loading periods");

    // ---------------------------------------------------------------------
    // Filter out future periods
    // ---------------------------------------------------------------------

    FilterUtils.filter(periods, new PastAndCurrentPeriodFilter());

    clock.logTime("Number of periods: " + periods.size());
    notifier.notify(id, DATAMART, "Filtering data elements without data");

    // ---------------------------------------------------------------------
    // Remove operands without data
    // ---------------------------------------------------------------------

    allOperands = dataMartManager.getOperandsWithData(allOperands);

    indicatorOperands.retainAll(allOperands);

    clock.logTime(
        "Number of operands with data: "
            + allOperands.size()
            + ", "
            + SystemUtils.getMemoryString());
    notifier.notify(id, DATAMART, "Populating crosstabulation table");

    // ---------------------------------------------------------------------
    // Create crosstabtable
    // ---------------------------------------------------------------------

    final Collection<Integer> intersectingPeriodIds =
        ConversionUtils.getIdentifiers(Period.class, periodService.getIntersectionPeriods(periods));
    final PeriodHierarchy periodHierarchy = periodService.getPeriodHierarchy(periods);
    final Set<Integer> orgUnitChildrenIds =
        organisationUnitService.getOrganisationUnitHierarchy().getChildren(organisationUnitIds);
    final List<Integer> crossTabOrgUnitIds = new ArrayList<Integer>(orgUnitChildrenIds);

    final String key = crossTabService.createCrossTabTable(crossTabOrgUnitIds);

    final List<DataElementOperand> operandList = new ArrayList<DataElementOperand>(allOperands);
    Collections.shuffle(operandList);

    final List<List<DataElementOperand>> operandPages =
        new PaginatedList<DataElementOperand>(operandList).setNumberOfPages(cpuCores).getPages();

    List<Future<?>> crossTabFutures = new ArrayList<Future<?>>();

    for (List<DataElementOperand> operandPage : operandPages) {
      crossTabFutures.add(
          crossTabService.populateCrossTabTable(
              operandPage, intersectingPeriodIds, crossTabOrgUnitIds, key));
    }

    ConcurrentUtils.waitForCompletion(crossTabFutures);

    clock.logTime("Populated crosstab table, " + SystemUtils.getMemoryString());
    notifier.notify(id, DATAMART, "Exporting data element data");

    final boolean isDataElements = true;

    final boolean isIndicators = indicators != null && indicators.size() > 0;

    final int groupLevel =
        (Integer)
            systemSettingManager.getSystemSetting(
                KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL);

    final boolean isGroups =
        organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0;

    if (isDataElements) {
      // -----------------------------------------------------------------
      // 1. Export data element values
      // -----------------------------------------------------------------

      if (operandList.size() > 0) {
        final OrganisationUnitHierarchy orgUnitHierarchy =
            organisationUnitService
                .getOrganisationUnitHierarchy()
                .prepareChildren(organisationUnits);

        List<Future<?>> futures = new ArrayList<Future<?>>();

        for (List<DataElementOperand> operandPage : operandPages) {
          futures.add(
              dataElementDataMart.exportDataValues(
                  operandPage,
                  periods,
                  organisationUnits,
                  null,
                  periodHierarchy,
                  orgUnitHierarchy,
                  AggregatedDataValueTempBatchHandler.class,
                  key));
        }

        ConcurrentUtils.waitForCompletion(futures);
      }

      clock.logTime(
          "Exported values for data element operands ("
              + operandList.size()
              + "), pages: "
              + operandPages.size()
              + ", "
              + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Dropping data element index");

      // -----------------------------------------------------------------
      // 2. Drop data element index
      // -----------------------------------------------------------------

      dataMartManager.dropDataValueIndex();

      clock.logTime("Dropped data element index");
      notifier.notify(id, DATAMART, "Deleting existing data element data");

      // -----------------------------------------------------------------
      // 3. Delete existing aggregated data values
      // -----------------------------------------------------------------

      dataMartManager.deleteAggregatedDataValues(periodIds);

      clock.logTime("Deleted existing data element data");
      notifier.notify(id, DATAMART, "Copying data element data from temporary table");

      // -----------------------------------------------------------------
      // 4. Copy aggregated data values from temporary table
      // -----------------------------------------------------------------

      dataMartManager.copyAggregatedDataValuesFromTemp();

      clock.logTime("Copied data element data from temporary table");
      notifier.notify(id, DATAMART, "Creating data element index");

      // -----------------------------------------------------------------
      // 5. Create data element index
      // -----------------------------------------------------------------

      dataMartManager.createDataValueIndex();

      clock.logTime("Created data element index");
    }

    if (isGroups && isDataElements) {
      // -----------------------------------------------------------------
      // 1. Export data element values
      // -----------------------------------------------------------------

      notifier.notify(id, DATAMART, "Exporting data element org unit data");

      Collection<OrganisationUnit> groupOrganisationUnits =
          new HashSet<OrganisationUnit>(organisationUnits);

      FilterUtils.filter(
          groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter(groupLevel));

      if (operandList.size() > 0) {
        final OrganisationUnitHierarchy orgUnitHierarchy =
            organisationUnitService
                .getOrganisationUnitHierarchy()
                .prepareChildren(groupOrganisationUnits, organisationUnitGroups);

        List<Future<?>> futures = new ArrayList<Future<?>>();

        for (List<DataElementOperand> operandPage : operandPages) {
          futures.add(
              dataElementDataMart.exportDataValues(
                  operandPage,
                  periods,
                  groupOrganisationUnits,
                  organisationUnitGroups,
                  periodHierarchy,
                  orgUnitHierarchy,
                  AggregatedOrgUnitDataValueTempBatchHandler.class,
                  key));
        }

        ConcurrentUtils.waitForCompletion(futures);
      }

      clock.logTime(
          "Exported values for data element operands ("
              + operandList.size()
              + "), pages: "
              + operandPages.size()
              + ", "
              + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Dropping data element data indexes");

      // -----------------------------------------------------------------
      // 2. Drop data element index
      // -----------------------------------------------------------------

      dataMartManager.dropOrgUnitDataValueIndex();

      clock.logTime("Dropped org unit data element index");
      notifier.notify(id, DATAMART, "Deleting existing org unit data element data");

      // -----------------------------------------------------------------
      // 3. Delete existing aggregated data values
      // -----------------------------------------------------------------

      dataMartManager.deleteAggregatedOrgUnitDataValues(periodIds);

      clock.logTime("Deleted existing aggregated org unit datavalues");
      notifier.notify(id, DATAMART, "Copying org unit data element data");

      // -----------------------------------------------------------------
      // 4. Copy aggregated org unit data values from temporary table
      // -----------------------------------------------------------------

      dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp();

      clock.logTime("Copied org unit data element data from temporary table");
      notifier.notify(id, DATAMART, "Creating org unit data element index");

      // -----------------------------------------------------------------
      // 5. Create org unit data element index
      // -----------------------------------------------------------------

      dataMartManager.createOrgUnitDataValueIndex();

      clock.logTime("Created org unit data element index");
      notifier.notify(id, DATAMART, "Exporting data for org unit indicator data");
    }

    crossTabService.dropCrossTabTable(key);

    List<List<OrganisationUnit>> organisationUnitPages =
        new PaginatedList<OrganisationUnit>(organisationUnits)
            .setNumberOfPages(cpuCores)
            .getPages();

    if (isIndicators) {
      // -----------------------------------------------------------------
      // 1. Create and populate aggregated data cache
      // -----------------------------------------------------------------

      notifier.notify(id, DATAMART, "Populating aggregated data cache");

      crossTabService.createAggregatedDataCache(indicatorOperands, key);

      List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();

      for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
        aggregatedDataCacheFutures.add(
            crossTabService.populateAggregatedDataCache(
                indicatorOperands, periods, organisationUnitPage, key));
      }

      ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);

      clock.logTime(
          "Created aggregated data cache, number of indicator operands: "
              + indicatorOperands.size());
      notifier.notify(id, DATAMART, "Exporting indicator data");

      // -----------------------------------------------------------------
      // 2. Export indicator values
      // -----------------------------------------------------------------

      List<Future<?>> futures = new ArrayList<Future<?>>();

      for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
        futures.add(
            indicatorDataMart.exportIndicatorValues(
                indicators,
                periods,
                organisationUnitPage,
                null,
                indicatorOperands,
                AggregatedIndicatorValueTempBatchHandler.class,
                key));
      }

      ConcurrentUtils.waitForCompletion(futures);

      clock.logTime(
          "Exported values for indicators ("
              + indicators.size()
              + "), pages: "
              + organisationUnitPages.size()
              + ", "
              + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Dropping indicator index");

      // -----------------------------------------------------------------
      // 3. Drop aggregated data cache and indicator index
      // -----------------------------------------------------------------

      crossTabService.dropAggregatedDataCache(key);
      dataMartManager.dropIndicatorValueIndex();

      clock.logTime("Dropped indicator index, " + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Deleting existing indicator data");

      // -----------------------------------------------------------------
      // 4. Delete existing aggregated indicator values
      // -----------------------------------------------------------------

      dataMartManager.deleteAggregatedIndicatorValues(periodIds);

      clock.logTime("Deleted existing indicator data");
      notifier.notify(id, DATAMART, "Copying indicator data from temporary table");

      // -----------------------------------------------------------------
      // 5. Copy aggregated data values from temporary table
      // -----------------------------------------------------------------

      dataMartManager.copyAggregatedIndicatorValuesFromTemp();

      clock.logTime("Copied indicator data from temporary table");
      notifier.notify(id, DATAMART, "Creating indicator index");

      // -----------------------------------------------------------------
      // 6. Create indicator index
      // -----------------------------------------------------------------

      dataMartManager.createIndicatorValueIndex();

      clock.logTime("Created indicator index");
    }

    if (isGroups && isIndicators) {
      // -----------------------------------------------------------------
      // 1. Create aggregated data cache
      // -----------------------------------------------------------------

      notifier.notify(id, DATAMART, "Populating aggregated data cache");

      crossTabService.createAggregatedOrgUnitDataCache(indicatorOperands, key);

      List<Future<?>> aggregatedDataCacheFutures = new ArrayList<Future<?>>();

      for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
        aggregatedDataCacheFutures.add(
            crossTabService.populateAggregatedOrgUnitDataCache(
                indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key));
      }

      ConcurrentUtils.waitForCompletion(aggregatedDataCacheFutures);

      clock.logTime("Created aggregated org unit data cache");
      notifier.notify(id, DATAMART, "Exporting org unit indicator data");

      // -----------------------------------------------------------------
      // 2. Export indicator values
      // -----------------------------------------------------------------

      List<Future<?>> futures = new ArrayList<Future<?>>();

      for (List<OrganisationUnit> organisationUnitPage : organisationUnitPages) {
        futures.add(
            indicatorDataMart.exportIndicatorValues(
                indicators,
                periods,
                organisationUnitPage,
                organisationUnitGroups,
                indicatorOperands,
                AggregatedOrgUnitIndicatorValueTempBatchHandler.class,
                key));
      }

      ConcurrentUtils.waitForCompletion(futures);

      clock.logTime(
          "Exported values for indicators ("
              + indicators.size()
              + "), pages: "
              + organisationUnitPages.size()
              + ", "
              + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Dropping org unit indicator index");

      // -----------------------------------------------------------------
      // 3. Drop aggregated data cache and indicator index
      // -----------------------------------------------------------------

      crossTabService.dropAggregatedOrgUnitDataCache(key);
      dataMartManager.dropOrgUnitIndicatorValueIndex();

      clock.logTime("Dropped org unit indicator index, " + SystemUtils.getMemoryString());
      notifier.notify(id, DATAMART, "Deleting existing org unit indicator data");

      // -----------------------------------------------------------------
      // 4. Delete existing aggregated indicator values
      // -----------------------------------------------------------------

      dataMartManager.deleteAggregatedOrgUnitIndicatorValues(periodIds);

      clock.logTime("Deleted existing aggregated org unit indicatorvalues");
      notifier.notify(id, DATAMART, "Copying org unit indicator data from temporary table");

      // -----------------------------------------------------------------
      // 5. Copy aggregated org unit indicator values from temp table
      // -----------------------------------------------------------------

      dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp();

      clock.logTime("Copied org unit indicator data from temporary table");
      notifier.notify(id, DATAMART, "Creating org unit indicator indexes");

      // -----------------------------------------------------------------
      // 6. Create org unit indicator index
      // -----------------------------------------------------------------

      dataMartManager.createOrgUnitIndicatorValueIndex();

      clock.logTime("Created org unit indicator index");
    }

    // ---------------------------------------------------------------------
    // Drop temporary tables
    // ---------------------------------------------------------------------

    dataMartManager.dropTempAggregatedTables();

    clock.logTime("Dropped crosstab table");
    clock.logTime("Data mart export process completed");
    notifier.notify(id, DATAMART, INFO, "Data mart process completed", true);
  }
  @Transactional
  public void matchObject(int importObjectId, int existingObjectId) {
    ImportObject importObject = importObjectStore.getImportObject(importObjectId);

    Object object = importObject.getObject();

    // ---------------------------------------------------------------------
    // Updates the name of the import object to the name of the existing
    // object.
    // ---------------------------------------------------------------------

    if (object.getClass().equals(DataElement.class)) {
      DataElement element = (DataElement) object;

      element.setName(dataElementService.getDataElement(existingObjectId).getName());
    } else if (object.getClass().equals(DataElementGroup.class)) {
      DataElementGroup group = (DataElementGroup) object;

      group.setName(dataElementService.getDataElementGroup(existingObjectId).getName());
    } else if (object.getClass().equals(DataElementGroupSet.class)) {
      DataElementGroupSet groupSet = (DataElementGroupSet) object;

      groupSet.setName(dataElementService.getDataElementGroupSet(existingObjectId).getName());
    } else if (object.getClass().equals(IndicatorType.class)) {
      IndicatorType type = (IndicatorType) object;

      type.setName(indicatorService.getIndicatorType(existingObjectId).getName());
    } else if (object.getClass().equals(Indicator.class)) {
      Indicator indicator = (Indicator) object;

      indicator.setName(indicatorService.getIndicator(existingObjectId).getName());
    } else if (object.getClass().equals(IndicatorGroup.class)) {
      IndicatorGroup group = (IndicatorGroup) object;

      group.setName(indicatorService.getIndicatorGroup(existingObjectId).getName());
    } else if (object.getClass().equals(IndicatorGroupSet.class)) {
      IndicatorGroupSet groupSet = (IndicatorGroupSet) object;

      groupSet.setName(indicatorService.getIndicatorGroupSet(existingObjectId).getName());
    } else if (object.getClass().equals(DataDictionary.class)) {
      DataDictionary dictionary = (DataDictionary) object;

      dictionary.setName(dataDictionaryService.getDataDictionary(existingObjectId).getName());
    } else if (object.getClass().equals(DataSet.class)) {
      DataSet dataSet = (DataSet) object;

      dataSet.setName(dataSetService.getDataSet(existingObjectId).getName());
    } else if (object.getClass().equals(OrganisationUnit.class)) {
      OrganisationUnit unit = (OrganisationUnit) object;

      unit.setName(organisationUnitService.getOrganisationUnit(existingObjectId).getName());
    } else if (object.getClass().equals(OrganisationUnitGroup.class)) {
      OrganisationUnitGroup group = (OrganisationUnitGroup) object;

      group.setName(
          organisationUnitGroupService.getOrganisationUnitGroup(existingObjectId).getName());
    } else if (object.getClass().equals(OrganisationUnitGroupSet.class)) {
      OrganisationUnitGroupSet groupSet = (OrganisationUnitGroupSet) object;

      groupSet.setName(
          organisationUnitGroupService.getOrganisationUnitGroupSet(existingObjectId).getName());
    } else if (object.getClass().equals(OrganisationUnitLevel.class)) {
      OrganisationUnitLevel level = (OrganisationUnitLevel) object;

      level.setName(organisationUnitService.getOrganisationUnitLevel(existingObjectId).getName());
    } else if (object.getClass().equals(ValidationRule.class)) {
      ValidationRule validationRule = (ValidationRule) object;

      validationRule.setName(validationRuleService.getValidationRule(existingObjectId).getName());
    } else if (object.getClass().equals(Report.class)) {
      Report report = (Report) object;

      report.setName(reportService.getReport(existingObjectId).getName());
    } else if (object.getClass().equals(ReportTable.class)) {
      ReportTable reportTable = (ReportTable) object;

      reportTable.setName(reportTableService.getReportTable(existingObjectId).getName());
    } else if (object.getClass().equals(Chart.class)) {
      Chart chart = (Chart) object;

      chart.setName(chartService.getChart(existingObjectId).getName());
    } else if (object.getClass().equals(DataValue.class)) {
      DataValue dataValue = (DataValue) object;

      dataValue =
          updateDataValue(
              dataValue,
              dataValueService.getDataValue(
                  dataValue.getSource(),
                  dataValue.getDataElement(),
                  dataValue.getPeriod(),
                  dataValue.getOptionCombo()));
    }

    // ---------------------------------------------------------------------
    // Sets the status of the import object to match, these objects will
    // later be ignored on import all but is needed for matching of
    // associations.
    // ---------------------------------------------------------------------

    importObject.setStatus(ImportObjectStatus.MATCH);

    importObjectStore.updateImportObject(importObject);
  }