Example #1
0
  private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(
      DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
    Operator<?> inputOp1 = operator.getFirstInput();
    Operator<?> inputOp2 = operator.getSecondInput();

    if (inputOp1 == null) {
      throw new InvalidProgramException(
          "The binary operation " + operator.getName() + " has no first input.");
    }
    if (inputOp2 == null) {
      throw new InvalidProgramException(
          "The binary operation " + operator.getName() + " has no second input.");
    }

    // compute inputs
    @SuppressWarnings("unchecked")
    List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
    @SuppressWarnings("unchecked")
    List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);

    @SuppressWarnings("unchecked")
    DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;

    // build the runtime context and compute broadcast variables, if necessary
    TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
    RuntimeUDFContext ctx;

    MetricGroup metrics = new UnregisteredMetricsGroup();

    if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
      ctx =
          superStep == 0
              ? new RuntimeUDFContext(
                  taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics)
              : new IterationRuntimeUDFContext(
                  taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);

      for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
        List<?> bcData = execute(bcInputs.getValue());
        ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
      }
    } else {
      ctx = null;
    }

    return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
  }
Example #2
0
    @Override
    public void postVisit(Operator<?> op) {

      if (op instanceof SingleInputOperator) {
        SingleInputOperator<?, ?, ?> siop = (SingleInputOperator<?, ?, ?>) op;

        if (dynamicPathOperations.contains(siop.getInput())) {
          dynamicPathOperations.add(op);
        } else {
          for (Operator<?> o : siop.getBroadcastInputs().values()) {
            if (dynamicPathOperations.contains(o)) {
              dynamicPathOperations.add(op);
              break;
            }
          }
        }
      } else if (op instanceof DualInputOperator) {
        DualInputOperator<?, ?, ?, ?> siop = (DualInputOperator<?, ?, ?, ?>) op;

        if (dynamicPathOperations.contains(siop.getFirstInput())) {
          dynamicPathOperations.add(op);
        } else if (dynamicPathOperations.contains(siop.getSecondInput())) {
          dynamicPathOperations.add(op);
        } else {
          for (Operator<?> o : siop.getBroadcastInputs().values()) {
            if (dynamicPathOperations.contains(o)) {
              dynamicPathOperations.add(op);
              break;
            }
          }
        }
      } else if (op.getClass() == PartialSolutionPlaceHolder.class
          || op.getClass() == WorksetPlaceHolder.class
          || op.getClass() == SolutionSetPlaceHolder.class) {
        dynamicPathOperations.add(op);
      } else if (op instanceof GenericDataSourceBase) {
        // skip
      } else {
        throw new RuntimeException("Cannot handle operator type " + op.getClass().getName());
      }
    }