Beispiel #1
0
      /**
       * Intermediate step of a CoGroup transformation. <br>
       * To continue the CoGroup transformation, provide a {@link
       * org.apache.flink.api.common.functions.RichCoGroupFunction} by calling {@link
       * org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.common.functions.CoGroupFunction)}.
       */
      private CoGroupOperatorWithoutFunction createCoGroupOperator(Keys<I2> keys2) {
        if (keys2 == null) {
          throw new NullPointerException();
        }

        if (keys2.isEmpty()) {
          throw new InvalidProgramException("The co-group keys must not be empty.");
        }
        try {
          keys1.areCompatible(keys2);
        } catch (IncompatibleKeysException ike) {
          throw new InvalidProgramException(
              "The pair of co-group keys are not compatible with each other.", ike);
        }

        return new CoGroupOperatorWithoutFunction(keys2);
      }
Beispiel #2
0
  @Override
  protected org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?>
      translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {

    String name = getName() != null ? getName() : "CoGroup at " + defaultName;
    try {
      keys1.areCompatible(keys2);
    } catch (IncompatibleKeysException e) {
      throw new InvalidProgramException("The types of the key fields do not match.", e);
    }

    final org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> po;

    if (keys1 instanceof SelectorFunctionKeys && keys2 instanceof SelectorFunctionKeys) {

      @SuppressWarnings("unchecked")
      SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
      @SuppressWarnings("unchecked")
      SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;

      po =
          translateSelectorFunctionCoGroup(
              selectorKeys1, selectorKeys2, function, getResultType(), name, input1, input2);

      po.setParallelism(getParallelism());
      po.setCustomPartitioner(customPartitioner);
    } else if (keys2 instanceof SelectorFunctionKeys) {

      int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();

      @SuppressWarnings("unchecked")
      SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;

      po =
          translateSelectorFunctionCoGroupRight(
              logicalKeyPositions1,
              selectorKeys2,
              function,
              getInput1Type(),
              getResultType(),
              name,
              input1,
              input2);

      po.setParallelism(getParallelism());
      po.setCustomPartitioner(customPartitioner);
    } else if (keys1 instanceof SelectorFunctionKeys) {

      @SuppressWarnings("unchecked")
      SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;

      int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();

      po =
          translateSelectorFunctionCoGroupLeft(
              selectorKeys1,
              logicalKeyPositions2,
              function,
              getInput2Type(),
              getResultType(),
              name,
              input1,
              input2);
    } else if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
      try {
        keys1.areCompatible(keys2);
      } catch (IncompatibleKeysException e) {
        throw new InvalidProgramException("The types of the key fields do not match.", e);
      }

      int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
      int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();

      CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> op =
          new CoGroupOperatorBase<>(
              function,
              new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()),
              logicalKeyPositions1,
              logicalKeyPositions2,
              name);

      op.setFirstInput(input1);
      op.setSecondInput(input2);
      po = op;
    } else {
      throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
    }

    // configure shared characteristics
    po.setParallelism(getParallelism());
    po.setCustomPartitioner(customPartitioner);

    if (groupSortKeyOrderFirst.size() > 0) {
      Ordering o = new Ordering();
      for (Pair<Integer, Order> entry : groupSortKeyOrderFirst) {
        o.appendOrdering(entry.getLeft(), null, entry.getRight());
      }
      po.setGroupOrderForInputOne(o);
    }
    if (groupSortKeyOrderSecond.size() > 0) {
      Ordering o = new Ordering();
      for (Pair<Integer, Order> entry : groupSortKeyOrderSecond) {
        o.appendOrdering(entry.getLeft(), null, entry.getRight());
      }
      po.setGroupOrderForInputTwo(o);
    }

    return po;
  }