protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) {
// select the name (or create a default one)
String name = this.name != null ? this.name : this.format.toString();
GenericDataSinkBase<T> sink =
new GenericDataSinkBase<T>(
this.format,
new UnaryOperatorInformation<T, Nothing>(this.type, new NothingTypeInfo()),
name);
// set input
sink.setInput(input);
// set parameters
if (this.parameters != null) {
sink.getParameters().addAll(this.parameters);
}
// set dop
if (this.dop > 0) {
// use specified dop
sink.setDegreeOfParallelism(this.dop);
} else {
// if no dop has been specified, use dop of input operator to enable chaining
sink.setDegreeOfParallelism(input.getDegreeOfParallelism());
}
if (this.sortKeyPositions != null) {
// configure output sorting
Ordering ordering = new Ordering();
for (int i = 0; i < this.sortKeyPositions.length; i++) {
ordering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]);
}
sink.setLocalOrder(ordering);
}
return sink;
}
public GroupReduceWithCombineProperties(
FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
super(groupKeys);
// if we have an additional ordering, construct the ordering to have primarily the grouping
// fields
if (additionalOrderKeys != null) {
this.ordering = new Ordering();
for (Integer key : this.keyList) {
this.ordering.appendOrdering(key, null, Order.ANY);
}
// and next the additional order fields
for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
Integer field = additionalOrderKeys.getFieldNumber(i);
Order order = additionalOrderKeys.getOrder(i);
this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
}
} else {
this.ordering = null;
}
this.customPartitioner = customPartitioner;
}
@Override
protected org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?>
translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
String name = getName() != null ? getName() : "CoGroup at " + defaultName;
try {
keys1.areCompatible(keys2);
} catch (IncompatibleKeysException e) {
throw new InvalidProgramException("The types of the key fields do not match.", e);
}
final org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> po;
if (keys1 instanceof SelectorFunctionKeys && keys2 instanceof SelectorFunctionKeys) {
@SuppressWarnings("unchecked")
SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
@SuppressWarnings("unchecked")
SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
po =
translateSelectorFunctionCoGroup(
selectorKeys1, selectorKeys2, function, getResultType(), name, input1, input2);
po.setParallelism(getParallelism());
po.setCustomPartitioner(customPartitioner);
} else if (keys2 instanceof SelectorFunctionKeys) {
int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
@SuppressWarnings("unchecked")
SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
po =
translateSelectorFunctionCoGroupRight(
logicalKeyPositions1,
selectorKeys2,
function,
getInput1Type(),
getResultType(),
name,
input1,
input2);
po.setParallelism(getParallelism());
po.setCustomPartitioner(customPartitioner);
} else if (keys1 instanceof SelectorFunctionKeys) {
@SuppressWarnings("unchecked")
SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
po =
translateSelectorFunctionCoGroupLeft(
selectorKeys1,
logicalKeyPositions2,
function,
getInput2Type(),
getResultType(),
name,
input1,
input2);
} else if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
try {
keys1.areCompatible(keys2);
} catch (IncompatibleKeysException e) {
throw new InvalidProgramException("The types of the key fields do not match.", e);
}
int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> op =
new CoGroupOperatorBase<>(
function,
new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()),
logicalKeyPositions1,
logicalKeyPositions2,
name);
op.setFirstInput(input1);
op.setSecondInput(input2);
po = op;
} else {
throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
}
// configure shared characteristics
po.setParallelism(getParallelism());
po.setCustomPartitioner(customPartitioner);
if (groupSortKeyOrderFirst.size() > 0) {
Ordering o = new Ordering();
for (Pair<Integer, Order> entry : groupSortKeyOrderFirst) {
o.appendOrdering(entry.getLeft(), null, entry.getRight());
}
po.setGroupOrderForInputOne(o);
}
if (groupSortKeyOrderSecond.size() > 0) {
Ordering o = new Ordering();
for (Pair<Integer, Order> entry : groupSortKeyOrderSecond) {
o.appendOrdering(entry.getLeft(), null, entry.getRight());
}
po.setGroupOrderForInputTwo(o);
}
return po;
}