protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) {
    // select the name (or create a default one)
    String name = this.name != null ? this.name : this.format.toString();
    GenericDataSinkBase<T> sink =
        new GenericDataSinkBase<T>(
            this.format,
            new UnaryOperatorInformation<T, Nothing>(this.type, new NothingTypeInfo()),
            name);
    // set input
    sink.setInput(input);
    // set parameters
    if (this.parameters != null) {
      sink.getParameters().addAll(this.parameters);
    }
    // set dop
    if (this.dop > 0) {
      // use specified dop
      sink.setDegreeOfParallelism(this.dop);
    } else {
      // if no dop has been specified, use dop of input operator to enable chaining
      sink.setDegreeOfParallelism(input.getDegreeOfParallelism());
    }

    if (this.sortKeyPositions != null) {
      // configure output sorting
      Ordering ordering = new Ordering();
      for (int i = 0; i < this.sortKeyPositions.length; i++) {
        ordering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]);
      }
      sink.setLocalOrder(ordering);
    }

    return sink;
  }
  public GroupReduceWithCombineProperties(
      FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
    super(groupKeys);

    // if we have an additional ordering, construct the ordering to have primarily the grouping
    // fields
    if (additionalOrderKeys != null) {
      this.ordering = new Ordering();
      for (Integer key : this.keyList) {
        this.ordering.appendOrdering(key, null, Order.ANY);
      }

      // and next the additional order fields
      for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
        Integer field = additionalOrderKeys.getFieldNumber(i);
        Order order = additionalOrderKeys.getOrder(i);
        this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
      }
    } else {
      this.ordering = null;
    }

    this.customPartitioner = customPartitioner;
  }
示例#3
0
  @Override
  protected org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?>
      translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {

    String name = getName() != null ? getName() : "CoGroup at " + defaultName;
    try {
      keys1.areCompatible(keys2);
    } catch (IncompatibleKeysException e) {
      throw new InvalidProgramException("The types of the key fields do not match.", e);
    }

    final org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> po;

    if (keys1 instanceof SelectorFunctionKeys && keys2 instanceof SelectorFunctionKeys) {

      @SuppressWarnings("unchecked")
      SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
      @SuppressWarnings("unchecked")
      SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;

      po =
          translateSelectorFunctionCoGroup(
              selectorKeys1, selectorKeys2, function, getResultType(), name, input1, input2);

      po.setParallelism(getParallelism());
      po.setCustomPartitioner(customPartitioner);
    } else if (keys2 instanceof SelectorFunctionKeys) {

      int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();

      @SuppressWarnings("unchecked")
      SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;

      po =
          translateSelectorFunctionCoGroupRight(
              logicalKeyPositions1,
              selectorKeys2,
              function,
              getInput1Type(),
              getResultType(),
              name,
              input1,
              input2);

      po.setParallelism(getParallelism());
      po.setCustomPartitioner(customPartitioner);
    } else if (keys1 instanceof SelectorFunctionKeys) {

      @SuppressWarnings("unchecked")
      SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;

      int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();

      po =
          translateSelectorFunctionCoGroupLeft(
              selectorKeys1,
              logicalKeyPositions2,
              function,
              getInput2Type(),
              getResultType(),
              name,
              input1,
              input2);
    } else if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
      try {
        keys1.areCompatible(keys2);
      } catch (IncompatibleKeysException e) {
        throw new InvalidProgramException("The types of the key fields do not match.", e);
      }

      int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
      int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();

      CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> op =
          new CoGroupOperatorBase<>(
              function,
              new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()),
              logicalKeyPositions1,
              logicalKeyPositions2,
              name);

      op.setFirstInput(input1);
      op.setSecondInput(input2);
      po = op;
    } else {
      throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
    }

    // configure shared characteristics
    po.setParallelism(getParallelism());
    po.setCustomPartitioner(customPartitioner);

    if (groupSortKeyOrderFirst.size() > 0) {
      Ordering o = new Ordering();
      for (Pair<Integer, Order> entry : groupSortKeyOrderFirst) {
        o.appendOrdering(entry.getLeft(), null, entry.getRight());
      }
      po.setGroupOrderForInputOne(o);
    }
    if (groupSortKeyOrderSecond.size() > 0) {
      Ordering o = new Ordering();
      for (Pair<Integer, Order> entry : groupSortKeyOrderSecond) {
        o.appendOrdering(entry.getLeft(), null, entry.getRight());
      }
      po.setGroupOrderForInputTwo(o);
    }

    return po;
  }