private static <IN, OUT, K>
PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionDistinct(
Keys.SelectorFunctionKeys<IN, ?> rawKeys,
RichGroupReduceFunction<IN, OUT> function,
TypeInformation<IN> inputType,
TypeInformation<OUT> outputType,
String name,
Operator<IN> input) {
@SuppressWarnings("unchecked")
final Keys.SelectorFunctionKeys<IN, K> keys = (Keys.SelectorFunctionKeys<IN, K>) rawKeys;
TypeInformation<Tuple2<K, IN>> typeInfoWithKey =
new TupleTypeInfo<Tuple2<K, IN>>(keys.getKeyType(), inputType);
KeyExtractingMapper<IN, K> extractor = new KeyExtractingMapper<IN, K>(keys.getKeyExtractor());
PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
new PlanUnwrappingReduceGroupOperator<IN, OUT, K>(
function, keys, name, outputType, typeInfoWithKey, true);
MapOperatorBase<IN, Tuple2<K, IN>, MapFunction<IN, Tuple2<K, IN>>> mapper =
new MapOperatorBase<IN, Tuple2<K, IN>, MapFunction<IN, Tuple2<K, IN>>>(
extractor,
new UnaryOperatorInformation<IN, Tuple2<K, IN>>(inputType, typeInfoWithKey),
"Key Extractor");
reducer.setInput(mapper);
mapper.setInput(input);
// set the mapper's parallelism to the input parallelism to make sure it is chained
mapper.setDegreeOfParallelism(input.getDegreeOfParallelism());
return reducer;
}
private static <T, K> MapOperatorBase<Tuple2<K, T>, T, ?> translateSelectorFunctionPartitioner(
Keys.SelectorFunctionKeys<T, ?> rawKeys,
PartitionMethod pMethod,
TypeInformation<T> inputType,
String name,
Operator<T> input,
int partitionDop,
Partitioner<?> customPartitioner) {
@SuppressWarnings("unchecked")
final Keys.SelectorFunctionKeys<T, K> keys = (Keys.SelectorFunctionKeys<T, K>) rawKeys;
TypeInformation<Tuple2<K, T>> typeInfoWithKey =
new TupleTypeInfo<Tuple2<K, T>>(keys.getKeyType(), inputType);
UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo =
new UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>>(typeInfoWithKey, typeInfoWithKey);
KeyExtractingMapper<T, K> extractor = new KeyExtractingMapper<T, K>(keys.getKeyExtractor());
MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>> keyExtractingMap =
new MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>>(
extractor,
new UnaryOperatorInformation<T, Tuple2<K, T>>(inputType, typeInfoWithKey),
"Key Extractor");
PartitionOperatorBase<Tuple2<K, T>> noop =
new PartitionOperatorBase<Tuple2<K, T>>(operatorInfo, pMethod, new int[] {0}, name);
MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> keyRemovingMap =
new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>(
new KeyRemovingMapper<T, K>(),
new UnaryOperatorInformation<Tuple2<K, T>, T>(typeInfoWithKey, inputType),
"Key Extractor");
keyExtractingMap.setInput(input);
noop.setInput(keyExtractingMap);
keyRemovingMap.setInput(noop);
noop.setCustomPartitioner(customPartitioner);
// set parallelism
keyExtractingMap.setParallelism(input.getParallelism());
noop.setParallelism(partitionDop);
keyRemovingMap.setParallelism(partitionDop);
return keyRemovingMap;
}
