@Override public void initialize() throws InstantiationException, IllegalAccessException { // If we run with separate master, we need to initialize the worker context // ourselves because a worker context is never created (and the worker context // is the one that initializes our configuration according to the user settings). WorkerContext workerContext = (WorkerContext) getConf().createWorkerContext(); workerContext.setConf(getConf()); Configuration conf = Configuration.get(); CommunicationStrategy communicationStrategy = conf.createCommunicationStrategy(conf, null, workerContext); numPhases = communicationStrategy.getNumPhases(); registeredAggregatorNames = new ArrayList<>(); savedAggregatorValues = new HashMap<>(); registerAggregator(AGG_EMBEDDINGS_GENERATED, LongSumAggregator.class); registerAggregator(AGG_EMBEDDINGS_PROCESSED, LongSumAggregator.class); registerAggregator(AGG_PROCESSED_SIZE_ODAG, LongMaxAggregator.class); registerAggregator(AGG_PROCESSED_SIZE_CACHE, LongSumAggregator.class); registerAggregator(AGG_CHILDREN_EVALUATED, LongSumAggregator.class); registerAggregator(AGG_EMBEDDINGS_OUTPUT, LongSumAggregator.class); Computation<?> computation = conf.createComputation(); computation.initAggregations(); Map<String, AggregationStorageMetadata> registeredAggregationStorages = conf.getAggregationsMetadata(); LOG.info("Registered aggregation storages: " + registeredAggregationStorages); for (Map.Entry<String, AggregationStorageMetadata> entry : registeredAggregationStorages.entrySet()) { String aggName = entry.getKey(); AggregationStorageMetadata aggStorageMetadata = entry.getValue(); registerAggregationStore(aggName, aggStorageMetadata); } masterComputation = conf.createMasterComputation(); masterComputation.setUnderlyingExecutionEngine(this); masterComputation.init(); }
protected void internalCompute() { DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); System.out.println("End of SS " + getSuperstep() + ":" + dateFormat.format(date)); System.out.println("Embeddings processed: " + getAggregatedValue(AGG_EMBEDDINGS_PROCESSED)); LongWritable processedSizeODAGWritable = getAggregatedValue(AGG_PROCESSED_SIZE_ODAG); long processedSizeODAG = processedSizeODAGWritable.get(); if (processedSizeODAG > 0) { System.out.println("Embeddings processed size (odag): " + processedSizeODAG); } LongWritable processedSizeCacheWritable = getAggregatedValue(AGG_PROCESSED_SIZE_CACHE); long processedSizeCache = processedSizeCacheWritable.get(); if (processedSizeCache > 0) { System.out.println("Embeddings processed size (cache): " + processedSizeCache); } System.out.println("Embeddings generated: " + getAggregatedValue(AGG_EMBEDDINGS_GENERATED)); System.out.println("Children evaluated: " + getAggregatedValue(AGG_CHILDREN_EVALUATED)); System.out.println("Embeddings output: " + getAggregatedValue(AGG_EMBEDDINGS_OUTPUT)); LongWritable numEmbeddingsProcessed = getAggregatedValue(AGG_EMBEDDINGS_PROCESSED); LongWritable numEmbeddingsGenerated = getAggregatedValue(AGG_EMBEDDINGS_GENERATED); // If we processed and generated no embeddings on the last superstep, execution has finished if (getSuperstep() > 0 && numEmbeddingsProcessed.get() == 0 && numEmbeddingsGenerated.get() == 0) { haltComputation(); } masterComputation.compute(); }