@Override
public void initialize() throws InstantiationException, IllegalAccessException {
// If we run with separate master, we need to initialize the worker context
// ourselves because a worker context is never created (and the worker context
// is the one that initializes our configuration according to the user settings).
WorkerContext workerContext = (WorkerContext) getConf().createWorkerContext();
workerContext.setConf(getConf());
Configuration conf = Configuration.get();
CommunicationStrategy communicationStrategy =
conf.createCommunicationStrategy(conf, null, workerContext);
numPhases = communicationStrategy.getNumPhases();
registeredAggregatorNames = new ArrayList<>();
savedAggregatorValues = new HashMap<>();
registerAggregator(AGG_EMBEDDINGS_GENERATED, LongSumAggregator.class);
registerAggregator(AGG_EMBEDDINGS_PROCESSED, LongSumAggregator.class);
registerAggregator(AGG_PROCESSED_SIZE_ODAG, LongMaxAggregator.class);
registerAggregator(AGG_PROCESSED_SIZE_CACHE, LongSumAggregator.class);
registerAggregator(AGG_CHILDREN_EVALUATED, LongSumAggregator.class);
registerAggregator(AGG_EMBEDDINGS_OUTPUT, LongSumAggregator.class);
Computation<?> computation = conf.createComputation();
computation.initAggregations();
Map<String, AggregationStorageMetadata> registeredAggregationStorages =
conf.getAggregationsMetadata();
LOG.info("Registered aggregation storages: " + registeredAggregationStorages);
for (Map.Entry<String, AggregationStorageMetadata> entry :
registeredAggregationStorages.entrySet()) {
String aggName = entry.getKey();
AggregationStorageMetadata aggStorageMetadata = entry.getValue();
registerAggregationStore(aggName, aggStorageMetadata);
}
masterComputation = conf.createMasterComputation();
masterComputation.setUnderlyingExecutionEngine(this);
masterComputation.init();
}
protected void internalCompute() {
DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
Date date = new Date();
System.out.println("End of SS " + getSuperstep() + ":" + dateFormat.format(date));
System.out.println("Embeddings processed: " + getAggregatedValue(AGG_EMBEDDINGS_PROCESSED));
LongWritable processedSizeODAGWritable = getAggregatedValue(AGG_PROCESSED_SIZE_ODAG);
long processedSizeODAG = processedSizeODAGWritable.get();
if (processedSizeODAG > 0) {
System.out.println("Embeddings processed size (odag): " + processedSizeODAG);
}
LongWritable processedSizeCacheWritable = getAggregatedValue(AGG_PROCESSED_SIZE_CACHE);
long processedSizeCache = processedSizeCacheWritable.get();
if (processedSizeCache > 0) {
System.out.println("Embeddings processed size (cache): " + processedSizeCache);
}
System.out.println("Embeddings generated: " + getAggregatedValue(AGG_EMBEDDINGS_GENERATED));
System.out.println("Children evaluated: " + getAggregatedValue(AGG_CHILDREN_EVALUATED));
System.out.println("Embeddings output: " + getAggregatedValue(AGG_EMBEDDINGS_OUTPUT));
LongWritable numEmbeddingsProcessed = getAggregatedValue(AGG_EMBEDDINGS_PROCESSED);
LongWritable numEmbeddingsGenerated = getAggregatedValue(AGG_EMBEDDINGS_GENERATED);
// If we processed and generated no embeddings on the last superstep, execution has finished
if (getSuperstep() > 0
&& numEmbeddingsProcessed.get() == 0
&& numEmbeddingsGenerated.get() == 0) {
haltComputation();
}
masterComputation.compute();
}