private void applyAggregationLevels(List<AnalyticsTable> tables) {
int maxLevels = organisationUnitService.getNumberOfOrganisationalLevels();
boolean hasAggLevels = false;
levelLoop:
for (int i = 0; i < maxLevels; i++) {
int level = maxLevels - i;
Collection<String> dataElements =
IdentifiableObjectUtils.getUids(
dataElementService.getDataElementsByAggregationLevel(level));
if (dataElements.isEmpty()) {
continue levelLoop;
}
hasAggLevels = true;
ConcurrentLinkedQueue<AnalyticsTable> tableQ = new ConcurrentLinkedQueue<>(tables);
List<Future<?>> futures = new ArrayList<>();
for (int j = 0; j < getProcessNo(); j++) {
futures.add(tableManager.applyAggregationLevels(tableQ, dataElements, level));
}
ConcurrentUtils.waitForCompletion(futures);
}
if (hasAggLevels) {
vacuumTables(tables);
log.info("Vacuumed tables");
}
}
@Override
public void update(Integer lastYears, TaskId taskId) {
int processNo = getProcessNo();
int orgUnitLevelNo = organisationUnitService.getNumberOfOrganisationalLevels();
Clock clock =
new Clock(log)
.startClock()
.logTime(
"Starting update, processes: "
+ processNo
+ ", org unit levels: "
+ orgUnitLevelNo);
String validState = tableManager.validState();
if (validState != null) {
notifier.notify(taskId, validState);
return;
}
Date earliest = PartitionUtils.getEarliestDate(lastYears);
final List<AnalyticsTable> tables = tableManager.getTables(earliest);
final String tableName = tableManager.getTableName();
clock.logTime(
"Table update start: "
+ tableName
+ ", processes: "
+ processNo
+ ", partitions: "
+ tables
+ ", last years: "
+ lastYears
+ ", earliest: "
+ earliest);
notifier.notify(
taskId,
"Performing pre-create table work, processes: "
+ processNo
+ ", org unit levels: "
+ orgUnitLevelNo);
tableManager.preCreateTables();
clock.logTime("Performed pre-create table work");
notifier.notify(taskId, "Creating analytics tables");
createTables(tables);
clock.logTime("Created analytics tables");
notifier.notify(taskId, "Populating analytics tables");
populateTables(tables);
clock.logTime("Populated analytics tables");
notifier.notify(taskId, "Applying aggregation levels");
applyAggregationLevels(tables);
clock.logTime("Applied aggregation levels");
notifier.notify(taskId, "Creating indexes");
createIndexes(tables);
clock.logTime("Created indexes");
notifier.notify(taskId, "Swapping analytics tables");
swapTables(tables, clock, taskId);
clock.logTime("Swapped tables");
notifier.notify(taskId, "Clearing caches");
partitionManager.clearCaches();
clock.logTime("Table update done: " + tableName);
notifier.notify(taskId, "Table update done");
}