@Test
public void testAllReduceDriverImmutableEmpty() {
try {
TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
context.setInput1(input, typeInfo.createSerializer());
context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
AllReduceDriver<Tuple2<String, Integer>> driver =
new AllReduceDriver<Tuple2<String, Integer>>();
driver.setup(context);
driver.prepare();
driver.run();
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testImmutableEmpty() {
try {
TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>(
1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
Collections.shuffle(data);
TupleTypeInfo<Tuple2<String, Integer>> typeInfo =
(TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
TypeComparator<Tuple2<String, Integer>> comparator =
typeInfo.createComparator(new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());
GatheringCollector<Tuple2<String, Integer>> result =
new GatheringCollector<Tuple2<String, Integer>>(
typeInfo.createSerializer(new ExecutionConfig()));
context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
context.setComparator1(comparator);
context.setCollector(result);
ReduceCombineDriver<Tuple2<String, Integer>> driver =
new ReduceCombineDriver<Tuple2<String, Integer>>();
driver.setup(context);
driver.prepare();
driver.run();
Assert.assertEquals(0, result.getList().size());
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testReduceDriverMutable() {
try {
{
TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>
context =
new TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(
1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo =
(TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input =
new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(
data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
TypeComparator<Tuple2<StringValue, IntValue>> comparator =
typeInfo.createComparator(
new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());
GatheringCollector<Tuple2<StringValue, IntValue>> result =
new GatheringCollector<Tuple2<StringValue, IntValue>>(
typeInfo.createSerializer(new ExecutionConfig()));
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
context.setComparator1(comparator);
context.setCollector(result);
context.setUdf(new ConcatSumFirstMutableReducer());
ReduceCombineDriver<Tuple2<StringValue, IntValue>> driver =
new ReduceCombineDriver<Tuple2<StringValue, IntValue>>();
driver.setup(context);
driver.prepare();
driver.run();
Object[] res = result.getList().toArray();
Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
DriverTestData.compareTupleArrays(expected, res);
}
{
TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>
context =
new TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(
1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo =
(TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input =
new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(
data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
TypeComparator<Tuple2<StringValue, IntValue>> comparator =
typeInfo.createComparator(
new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());
GatheringCollector<Tuple2<StringValue, IntValue>> result =
new GatheringCollector<Tuple2<StringValue, IntValue>>(
typeInfo.createSerializer(new ExecutionConfig()));
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
context.setComparator1(comparator);
context.setCollector(result);
context.setUdf(new ConcatSumSecondMutableReducer());
ReduceCombineDriver<Tuple2<StringValue, IntValue>> driver =
new ReduceCombineDriver<Tuple2<StringValue, IntValue>>();
driver.setup(context);
driver.prepare();
driver.run();
Object[] res = result.getList().toArray();
Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
DriverTestData.compareTupleArrays(expected, res);
}
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testAllReduceDriverImmutable() {
try {
{
TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input =
new RegularToMutableObjectIterator<Tuple2<String, Integer>>(
data.iterator(), typeInfo.createSerializer());
GatheringCollector<Tuple2<String, Integer>> result =
new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer());
context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
context.setInput1(input, typeInfo.createSerializer());
context.setCollector(result);
context.setUdf(new ConcatSumFirstReducer());
AllReduceDriver<Tuple2<String, Integer>> driver =
new AllReduceDriver<Tuple2<String, Integer>>();
driver.setup(context);
driver.prepare();
driver.run();
Tuple2<String, Integer> res = result.getList().get(0);
char[] foundString = res.f0.toCharArray();
Arrays.sort(foundString);
char[] expectedString = "abcddeeeffff".toCharArray();
Arrays.sort(expectedString);
Assert.assertArrayEquals(expectedString, foundString);
Assert.assertEquals(78, res.f1.intValue());
}
{
TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
new TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input =
new RegularToMutableObjectIterator<Tuple2<String, Integer>>(
data.iterator(), typeInfo.createSerializer());
GatheringCollector<Tuple2<String, Integer>> result =
new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer());
context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
context.setInput1(input, typeInfo.createSerializer());
context.setCollector(result);
context.setUdf(new ConcatSumSecondReducer());
AllReduceDriver<Tuple2<String, Integer>> driver =
new AllReduceDriver<Tuple2<String, Integer>>();
driver.setup(context);
driver.prepare();
driver.run();
Tuple2<String, Integer> res = result.getList().get(0);
char[] foundString = res.f0.toCharArray();
Arrays.sort(foundString);
char[] expectedString = "abcddeeeffff".toCharArray();
Arrays.sort(expectedString);
Assert.assertArrayEquals(expectedString, foundString);
Assert.assertEquals(78, res.f1.intValue());
}
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
