@Override
public void dropTables() {
List<AnalyticsTable> tables = tableManager.getAllTables();
for (AnalyticsTable table : tables) {
tableManager.dropTable(table.getTableName());
tableManager.dropTable(table.getTempTableName());
}
}
private void vacuumTables(List<AnalyticsTable> tables) {
ConcurrentLinkedQueue<AnalyticsTable> tableQ = new ConcurrentLinkedQueue<>(tables);
List<Future<?>> futures = new ArrayList<>();
for (int i = 0; i < getProcessNo(); i++) {
tableManager.vacuumTablesAsync(tableQ);
}
ConcurrentUtils.waitForCompletion(futures);
}
private void swapTables(List<AnalyticsTable> tables, Clock clock, TaskId taskId) {
resourceTableService.dropAllSqlViews();
clock.logTime("Dropped SQL views");
notifier.notify(taskId, "Swapping tables");
for (AnalyticsTable table : tables) {
tableManager.swapTable(table);
}
clock.logTime("Swapped tables");
notifier.notify(taskId, "Creating SQL views");
resourceTableService.createAllSqlViews();
}
private void populateTables(List<AnalyticsTable> tables) {
int taskNo = Math.min(getProcessNo(), tables.size());
log.info("Populate table task number: " + taskNo);
ConcurrentLinkedQueue<AnalyticsTable> tableQ = new ConcurrentLinkedQueue<>(tables);
List<Future<?>> futures = new ArrayList<>();
for (int i = 0; i < taskNo; i++) {
futures.add(tableManager.populateTableAsync(tableQ));
}
ConcurrentUtils.waitForCompletion(futures);
}
private void createIndexes(List<AnalyticsTable> tables) {
ConcurrentLinkedQueue<AnalyticsIndex> indexes = new ConcurrentLinkedQueue<>();
for (AnalyticsTable table : tables) {
List<String[]> columns = table.getDimensionColumns();
for (String[] column : columns) {
indexes.add(new AnalyticsIndex(table.getTempTableName(), column[0]));
}
}
log.info("No of analytics table indexes: " + indexes.size());
List<Future<?>> futures = new ArrayList<>();
for (int i = 0; i < getProcessNo(); i++) {
futures.add(tableManager.createIndexesAsync(indexes));
}
ConcurrentUtils.waitForCompletion(futures);
}
private void applyAggregationLevels(List<AnalyticsTable> tables) {
int maxLevels = organisationUnitService.getNumberOfOrganisationalLevels();
boolean hasAggLevels = false;
levelLoop:
for (int i = 0; i < maxLevels; i++) {
int level = maxLevels - i;
Collection<String> dataElements =
IdentifiableObjectUtils.getUids(
dataElementService.getDataElementsByAggregationLevel(level));
if (dataElements.isEmpty()) {
continue levelLoop;
}
hasAggLevels = true;
ConcurrentLinkedQueue<AnalyticsTable> tableQ = new ConcurrentLinkedQueue<>(tables);
List<Future<?>> futures = new ArrayList<>();
for (int j = 0; j < getProcessNo(); j++) {
futures.add(tableManager.applyAggregationLevels(tableQ, dataElements, level));
}
ConcurrentUtils.waitForCompletion(futures);
}
if (hasAggLevels) {
vacuumTables(tables);
log.info("Vacuumed tables");
}
}
@Override
public void update(Integer lastYears, TaskId taskId) {
int processNo = getProcessNo();
int orgUnitLevelNo = organisationUnitService.getNumberOfOrganisationalLevels();
Clock clock =
new Clock(log)
.startClock()
.logTime(
"Starting update, processes: "
+ processNo
+ ", org unit levels: "
+ orgUnitLevelNo);
String validState = tableManager.validState();
if (validState != null) {
notifier.notify(taskId, validState);
return;
}
Date earliest = PartitionUtils.getEarliestDate(lastYears);
final List<AnalyticsTable> tables = tableManager.getTables(earliest);
final String tableName = tableManager.getTableName();
clock.logTime(
"Table update start: "
+ tableName
+ ", processes: "
+ processNo
+ ", partitions: "
+ tables
+ ", last years: "
+ lastYears
+ ", earliest: "
+ earliest);
notifier.notify(
taskId,
"Performing pre-create table work, processes: "
+ processNo
+ ", org unit levels: "
+ orgUnitLevelNo);
tableManager.preCreateTables();
clock.logTime("Performed pre-create table work");
notifier.notify(taskId, "Creating analytics tables");
createTables(tables);
clock.logTime("Created analytics tables");
notifier.notify(taskId, "Populating analytics tables");
populateTables(tables);
clock.logTime("Populated analytics tables");
notifier.notify(taskId, "Applying aggregation levels");
applyAggregationLevels(tables);
clock.logTime("Applied aggregation levels");
notifier.notify(taskId, "Creating indexes");
createIndexes(tables);
clock.logTime("Created indexes");
notifier.notify(taskId, "Swapping analytics tables");
swapTables(tables, clock, taskId);
clock.logTime("Swapped tables");
notifier.notify(taskId, "Clearing caches");
partitionManager.clearCaches();
clock.logTime("Table update done: " + tableName);
notifier.notify(taskId, "Table update done");
}
private void createTables(List<AnalyticsTable> tables) {
for (AnalyticsTable table : tables) {
tableManager.createTable(table);
}
}