@Test
  public void testCorrectTranslation() {
    try {
      final String JOB_NAME = "Test JobName";
      final String ITERATION_NAME = "Test Name";

      final String BEFORE_NEXT_WORKSET_MAP = "Some Mapper";

      final String AGGREGATOR_NAME = "AggregatorName";

      final int[] ITERATION_KEYS = new int[] {2};
      final int NUM_ITERATIONS = 13;

      final int DEFAULT_DOP = 133;
      final int ITERATION_DOP = 77;

      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

      // ------------ construct the test program ------------------
      {
        env.setDegreeOfParallelism(DEFAULT_DOP);

        @SuppressWarnings("unchecked")
        DataSet<Tuple3<Double, Long, String>> initialSolutionSet =
            env.fromElements(new Tuple3<Double, Long, String>(3.44, 5L, "abc"));

        @SuppressWarnings("unchecked")
        DataSet<Tuple2<Double, String>> initialWorkSet =
            env.fromElements(new Tuple2<Double, String>(1.23, "abc"));

        DeltaIteration<Tuple3<Double, Long, String>, Tuple2<Double, String>> iteration =
            initialSolutionSet.iterateDelta(initialWorkSet, NUM_ITERATIONS, ITERATION_KEYS);
        iteration.name(ITERATION_NAME).parallelism(ITERATION_DOP);

        iteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());

        // test that multiple workset consumers are supported
        DataSet<Tuple2<Double, String>> worksetSelfJoin =
            iteration
                .getWorkset()
                .map(new IdentityMapper<Tuple2<Double, String>>())
                .join(iteration.getWorkset())
                .where(1)
                .equalTo(1)
                .projectFirst(0, 1)
                .types(Double.class, String.class);

        DataSet<Tuple3<Double, Long, String>> joined =
            worksetSelfJoin
                .join(iteration.getSolutionSet())
                .where(1)
                .equalTo(2)
                .with(new SolutionWorksetJoin());

        DataSet<Tuple3<Double, Long, String>> result =
            iteration.closeWith(
                joined, joined.map(new NextWorksetMapper()).name(BEFORE_NEXT_WORKSET_MAP));

        result.print();
        result.writeAsText("/dev/null");
      }

      Plan p = env.createProgramPlan(JOB_NAME);

      // ------------- validate the plan ----------------
      assertEquals(JOB_NAME, p.getJobName());
      assertEquals(DEFAULT_DOP, p.getDefaultParallelism());

      // validate the iteration
      GenericDataSinkBase<?> sink1, sink2;
      {
        Iterator<? extends GenericDataSinkBase<?>> sinks = p.getDataSinks().iterator();
        sink1 = sinks.next();
        sink2 = sinks.next();
      }

      DeltaIterationBase<?, ?> iteration = (DeltaIterationBase<?, ?>) sink1.getInput();

      // check that multi consumer translation works for iterations
      assertEquals(iteration, sink2.getInput());

      // check the basic iteration properties
      assertEquals(NUM_ITERATIONS, iteration.getMaximumNumberOfIterations());
      assertArrayEquals(ITERATION_KEYS, iteration.getSolutionSetKeyFields());
      assertEquals(ITERATION_DOP, iteration.getDegreeOfParallelism());
      assertEquals(ITERATION_NAME, iteration.getName());

      MapOperatorBase<?, ?, ?> nextWorksetMapper =
          (MapOperatorBase<?, ?, ?>) iteration.getNextWorkset();
      JoinOperatorBase<?, ?, ?, ?> solutionSetJoin =
          (JoinOperatorBase<?, ?, ?, ?>) iteration.getSolutionSetDelta();
      JoinOperatorBase<?, ?, ?, ?> worksetSelfJoin =
          (JoinOperatorBase<?, ?, ?, ?>) solutionSetJoin.getFirstInput();
      MapOperatorBase<?, ?, ?> worksetMapper =
          (MapOperatorBase<?, ?, ?>) worksetSelfJoin.getFirstInput();

      assertEquals(IdentityMapper.class, worksetMapper.getUserCodeWrapper().getUserCodeClass());
      assertEquals(
          NextWorksetMapper.class, nextWorksetMapper.getUserCodeWrapper().getUserCodeClass());
      if (solutionSetJoin.getUserCodeWrapper().getUserCodeObject() instanceof WrappingFunction) {
        WrappingFunction<?> wf =
            (WrappingFunction<?>) solutionSetJoin.getUserCodeWrapper().getUserCodeObject();
        assertEquals(SolutionWorksetJoin.class, wf.getWrappedFunction().getClass());
      } else {
        assertEquals(
            SolutionWorksetJoin.class, solutionSetJoin.getUserCodeWrapper().getUserCodeClass());
      }

      assertEquals(BEFORE_NEXT_WORKSET_MAP, nextWorksetMapper.getName());

      assertEquals(
          AGGREGATOR_NAME,
          iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
    } catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
예제 #2
0
  @SuppressWarnings("unchecked")
  private <T> List<T> executeDeltaIteration(DeltaIterationBase<?, ?> iteration) throws Exception {
    Operator<?> solutionInput = iteration.getInitialSolutionSet();
    Operator<?> worksetInput = iteration.getInitialWorkset();
    if (solutionInput == null) {
      throw new InvalidProgramException(
          "The delta iteration " + iteration.getName() + " has no initial solution set.");
    }
    if (worksetInput == null) {
      throw new InvalidProgramException(
          "The delta iteration " + iteration.getName() + " has no initial workset.");
    }
    if (iteration.getSolutionSetDelta() == null) {
      throw new InvalidProgramException(
          "The iteration "
              + iteration.getName()
              + " has no solution set delta defined (is not closed).");
    }
    if (iteration.getNextWorkset() == null) {
      throw new InvalidProgramException(
          "The iteration " + iteration.getName() + " has no workset defined (is not closed).");
    }

    List<T> solutionInputData = (List<T>) execute(solutionInput);
    List<T> worksetInputData = (List<T>) execute(worksetInput);

    // get the operators that are iterative
    Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
    DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
    iteration.getSolutionSetDelta().accept(dynCollector);
    iteration.getNextWorkset().accept(dynCollector);

    BinaryOperatorInformation<?, ?, ?> operatorInfo = iteration.getOperatorInfo();
    TypeInformation<?> solutionType = operatorInfo.getFirstInputType();

    int[] keyColumns = iteration.getSolutionSetKeyFields();
    boolean[] inputOrderings = new boolean[keyColumns.length];
    TypeComparator<T> inputComparator =
        ((CompositeType<T>) solutionType)
            .createComparator(keyColumns, inputOrderings, 0, executionConfig);

    Map<TypeComparable<T>, T> solutionMap =
        new HashMap<TypeComparable<T>, T>(solutionInputData.size());
    // fill the solution from the initial input
    for (T delta : solutionInputData) {
      TypeComparable<T> wrapper = new TypeComparable<T>(delta, inputComparator);
      solutionMap.put(wrapper, delta);
    }

    List<?> currentWorkset = worksetInputData;

    // register the aggregators
    for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
      aggregators.put(a.getName(), a.getAggregator());
    }

    String convCriterionAggName =
        iteration.getAggregators().getConvergenceCriterionAggregatorName();
    ConvergenceCriterion<Value> convCriterion =
        (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();

    final int maxIterations = iteration.getMaximumNumberOfIterations();

    for (int superstep = 1; superstep <= maxIterations; superstep++) {

      List<T> currentSolution = new ArrayList<T>(solutionMap.size());
      currentSolution.addAll(solutionMap.values());

      // set the input to the current partial solution
      this.intermediateResults.put(iteration.getSolutionSet(), currentSolution);
      this.intermediateResults.put(iteration.getWorkset(), currentWorkset);

      // set the superstep number
      iterationSuperstep = superstep;

      // grab the current iteration result
      List<T> solutionSetDelta = (List<T>) execute(iteration.getSolutionSetDelta(), superstep);
      this.intermediateResults.put(iteration.getSolutionSetDelta(), solutionSetDelta);

      // update the solution
      for (T delta : solutionSetDelta) {
        TypeComparable<T> wrapper = new TypeComparable<T>(delta, inputComparator);
        solutionMap.put(wrapper, delta);
      }

      currentWorkset = execute(iteration.getNextWorkset(), superstep);

      if (currentWorkset.isEmpty()) {
        break;
      }

      // evaluate the aggregator convergence criterion
      if (convCriterion != null && convCriterionAggName != null) {
        Value v = aggregators.get(convCriterionAggName).getAggregate();
        if (convCriterion.isConverged(superstep, v)) {
          break;
        }
      }

      // clear the dynamic results
      for (Operator<?> o : dynamics) {
        intermediateResults.remove(o);
      }

      // set the previous iteration's aggregates and reset the aggregators
      for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
        previousAggregates.put(e.getKey(), e.getValue().getAggregate());
        e.getValue().reset();
      }
    }

    previousAggregates.clear();
    aggregators.clear();

    List<T> currentSolution = new ArrayList<T>(solutionMap.size());
    currentSolution.addAll(solutionMap.values());
    return currentSolution;
  }