示例#1
0
  @Override
  public final void compute() {
    inboundAggregationStorages.clear();

    if (numPhases == 1) {
      internalCompute();
    } else {
      currentPhase = (int) (super.getSuperstep() % numPhases);

      LOG.info("MASTER: Real SS" + super.getSuperstep());
      LOG.info("MASTER: Fake SS" + getSuperstep());
      LOG.info("MASTER: Phase" + currentPhase);

      if (currentPhase == 1) {
        internalCompute();

        for (String registeredAggregatorName : registeredAggregatorNames) {
          Writable value = super.getAggregatedValue(registeredAggregatorName);
          savedAggregatorValues.put(registeredAggregatorName, value);
          setAggregatedValue(registeredAggregatorName, null);
        }
      } else if (currentPhase == 0) {
        for (Map.Entry<String, Writable> savedAggregatorValuesEntry :
            savedAggregatorValues.entrySet()) {
          setAggregatedValue(
              savedAggregatorValuesEntry.getKey(), savedAggregatorValuesEntry.getValue());
        }

        savedAggregatorValues.clear();
      }
    }
  }
示例#2
0
  @Override
  public void initialize() throws InstantiationException, IllegalAccessException {
    // If we run with separate master, we need to initialize the worker context
    // ourselves because a worker context is never created (and the worker context
    // is the one that initializes our configuration according to the user settings).
    WorkerContext workerContext = (WorkerContext) getConf().createWorkerContext();
    workerContext.setConf(getConf());

    Configuration conf = Configuration.get();

    CommunicationStrategy communicationStrategy =
        conf.createCommunicationStrategy(conf, null, workerContext);

    numPhases = communicationStrategy.getNumPhases();

    registeredAggregatorNames = new ArrayList<>();
    savedAggregatorValues = new HashMap<>();

    registerAggregator(AGG_EMBEDDINGS_GENERATED, LongSumAggregator.class);
    registerAggregator(AGG_EMBEDDINGS_PROCESSED, LongSumAggregator.class);
    registerAggregator(AGG_PROCESSED_SIZE_ODAG, LongMaxAggregator.class);
    registerAggregator(AGG_PROCESSED_SIZE_CACHE, LongSumAggregator.class);
    registerAggregator(AGG_CHILDREN_EVALUATED, LongSumAggregator.class);
    registerAggregator(AGG_EMBEDDINGS_OUTPUT, LongSumAggregator.class);

    Computation<?> computation = conf.createComputation();

    computation.initAggregations();

    Map<String, AggregationStorageMetadata> registeredAggregationStorages =
        conf.getAggregationsMetadata();

    LOG.info("Registered aggregation storages: " + registeredAggregationStorages);

    for (Map.Entry<String, AggregationStorageMetadata> entry :
        registeredAggregationStorages.entrySet()) {
      String aggName = entry.getKey();
      AggregationStorageMetadata aggStorageMetadata = entry.getValue();

      registerAggregationStore(aggName, aggStorageMetadata);
    }

    masterComputation = conf.createMasterComputation();
    masterComputation.setUnderlyingExecutionEngine(this);
    masterComputation.init();
  }