コード例 #1
0
  @Override
  public void initialize() throws InstantiationException, IllegalAccessException {
    // If we run with separate master, we need to initialize the worker context
    // ourselves because a worker context is never created (and the worker context
    // is the one that initializes our configuration according to the user settings).
    WorkerContext workerContext = (WorkerContext) getConf().createWorkerContext();
    workerContext.setConf(getConf());

    Configuration conf = Configuration.get();

    CommunicationStrategy communicationStrategy =
        conf.createCommunicationStrategy(conf, null, workerContext);

    numPhases = communicationStrategy.getNumPhases();

    registeredAggregatorNames = new ArrayList<>();
    savedAggregatorValues = new HashMap<>();

    registerAggregator(AGG_EMBEDDINGS_GENERATED, LongSumAggregator.class);
    registerAggregator(AGG_EMBEDDINGS_PROCESSED, LongSumAggregator.class);
    registerAggregator(AGG_PROCESSED_SIZE_ODAG, LongMaxAggregator.class);
    registerAggregator(AGG_PROCESSED_SIZE_CACHE, LongSumAggregator.class);
    registerAggregator(AGG_CHILDREN_EVALUATED, LongSumAggregator.class);
    registerAggregator(AGG_EMBEDDINGS_OUTPUT, LongSumAggregator.class);

    Computation<?> computation = conf.createComputation();

    computation.initAggregations();

    Map<String, AggregationStorageMetadata> registeredAggregationStorages =
        conf.getAggregationsMetadata();

    LOG.info("Registered aggregation storages: " + registeredAggregationStorages);

    for (Map.Entry<String, AggregationStorageMetadata> entry :
        registeredAggregationStorages.entrySet()) {
      String aggName = entry.getKey();
      AggregationStorageMetadata aggStorageMetadata = entry.getValue();

      registerAggregationStore(aggName, aggStorageMetadata);
    }

    masterComputation = conf.createMasterComputation();
    masterComputation.setUnderlyingExecutionEngine(this);
    masterComputation.init();
  }
コード例 #2
0
  protected void internalCompute() {
    DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
    Date date = new Date();
    System.out.println("End of SS " + getSuperstep() + ":" + dateFormat.format(date));
    System.out.println("Embeddings processed: " + getAggregatedValue(AGG_EMBEDDINGS_PROCESSED));

    LongWritable processedSizeODAGWritable = getAggregatedValue(AGG_PROCESSED_SIZE_ODAG);
    long processedSizeODAG = processedSizeODAGWritable.get();

    if (processedSizeODAG > 0) {
      System.out.println("Embeddings processed size (odag): " + processedSizeODAG);
    }

    LongWritable processedSizeCacheWritable = getAggregatedValue(AGG_PROCESSED_SIZE_CACHE);
    long processedSizeCache = processedSizeCacheWritable.get();

    if (processedSizeCache > 0) {
      System.out.println("Embeddings processed size (cache): " + processedSizeCache);
    }

    System.out.println("Embeddings generated: " + getAggregatedValue(AGG_EMBEDDINGS_GENERATED));
    System.out.println("Children evaluated: " + getAggregatedValue(AGG_CHILDREN_EVALUATED));
    System.out.println("Embeddings output: " + getAggregatedValue(AGG_EMBEDDINGS_OUTPUT));

    LongWritable numEmbeddingsProcessed = getAggregatedValue(AGG_EMBEDDINGS_PROCESSED);
    LongWritable numEmbeddingsGenerated = getAggregatedValue(AGG_EMBEDDINGS_GENERATED);

    // If we processed and generated no embeddings on the last superstep, execution has finished
    if (getSuperstep() > 0
        && numEmbeddingsProcessed.get() == 0
        && numEmbeddingsGenerated.get() == 0) {
      haltComputation();
    }

    masterComputation.compute();
  }