public CoGroupOperator(
DataSet<I1> input1,
DataSet<I2> input2,
Keys<I1> keys1,
Keys<I2> keys2,
CoGroupFunction<I1, I2, OUT> function,
TypeInformation<OUT> returnType,
List<Pair<Integer, Order>> groupSortKeyOrderFirst,
List<Pair<Integer, Order>> groupSortKeyOrderSecond,
Partitioner<?> customPartitioner,
String defaultName) {
super(input1, input2, returnType);
this.function = function;
this.customPartitioner = customPartitioner;
this.defaultName = defaultName;
this.groupSortKeyOrderFirst =
groupSortKeyOrderFirst == null
? Collections.<Pair<Integer, Order>>emptyList()
: groupSortKeyOrderFirst;
this.groupSortKeyOrderSecond =
groupSortKeyOrderSecond == null
? Collections.<Pair<Integer, Order>>emptyList()
: groupSortKeyOrderSecond;
if (keys1 == null || keys2 == null) {
throw new NullPointerException();
}
// sanity check solution set key mismatches
if (input1 instanceof SolutionSetPlaceHolder) {
if (keys1 instanceof ExpressionKeys) {
int[] positions = keys1.computeLogicalKeyPositions();
((SolutionSetPlaceHolder<?>) input1).checkJoinKeyFields(positions);
} else {
throw new InvalidProgramException(
"Currently, the solution set may only be CoGrouped with using tuple field positions.");
}
}
if (input2 instanceof SolutionSetPlaceHolder) {
if (keys2 instanceof ExpressionKeys) {
int[] positions = keys2.computeLogicalKeyPositions();
((SolutionSetPlaceHolder<?>) input2).checkJoinKeyFields(positions);
} else {
throw new InvalidProgramException(
"Currently, the solution set may only be CoGrouped with using tuple field positions.");
}
}
this.keys1 = keys1;
this.keys2 = keys2;
UdfOperatorUtils.analyzeDualInputUdf(
this, CoGroupFunction.class, defaultName, function, keys1, keys2);
}
@Override
protected org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?>
translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
String name = getName() != null ? getName() : "CoGroup at " + defaultName;
try {
keys1.areCompatible(keys2);
} catch (IncompatibleKeysException e) {
throw new InvalidProgramException("The types of the key fields do not match.", e);
}
final org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> po;
if (keys1 instanceof SelectorFunctionKeys && keys2 instanceof SelectorFunctionKeys) {
@SuppressWarnings("unchecked")
SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
@SuppressWarnings("unchecked")
SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
po =
translateSelectorFunctionCoGroup(
selectorKeys1, selectorKeys2, function, getResultType(), name, input1, input2);
po.setParallelism(getParallelism());
po.setCustomPartitioner(customPartitioner);
} else if (keys2 instanceof SelectorFunctionKeys) {
int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
@SuppressWarnings("unchecked")
SelectorFunctionKeys<I2, ?> selectorKeys2 = (SelectorFunctionKeys<I2, ?>) keys2;
po =
translateSelectorFunctionCoGroupRight(
logicalKeyPositions1,
selectorKeys2,
function,
getInput1Type(),
getResultType(),
name,
input1,
input2);
po.setParallelism(getParallelism());
po.setCustomPartitioner(customPartitioner);
} else if (keys1 instanceof SelectorFunctionKeys) {
@SuppressWarnings("unchecked")
SelectorFunctionKeys<I1, ?> selectorKeys1 = (SelectorFunctionKeys<I1, ?>) keys1;
int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
po =
translateSelectorFunctionCoGroupLeft(
selectorKeys1,
logicalKeyPositions2,
function,
getInput2Type(),
getResultType(),
name,
input1,
input2);
} else if (keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys) {
try {
keys1.areCompatible(keys2);
} catch (IncompatibleKeysException e) {
throw new InvalidProgramException("The types of the key fields do not match.", e);
}
int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> op =
new CoGroupOperatorBase<>(
function,
new BinaryOperatorInformation<>(getInput1Type(), getInput2Type(), getResultType()),
logicalKeyPositions1,
logicalKeyPositions2,
name);
op.setFirstInput(input1);
op.setSecondInput(input2);
po = op;
} else {
throw new UnsupportedOperationException("Unrecognized or incompatible key types.");
}
// configure shared characteristics
po.setParallelism(getParallelism());
po.setCustomPartitioner(customPartitioner);
if (groupSortKeyOrderFirst.size() > 0) {
Ordering o = new Ordering();
for (Pair<Integer, Order> entry : groupSortKeyOrderFirst) {
o.appendOrdering(entry.getLeft(), null, entry.getRight());
}
po.setGroupOrderForInputOne(o);
}
if (groupSortKeyOrderSecond.size() > 0) {
Ordering o = new Ordering();
for (Pair<Integer, Order> entry : groupSortKeyOrderSecond) {
o.appendOrdering(entry.getLeft(), null, entry.getRight());
}
po.setGroupOrderForInputTwo(o);
}
return po;
}