@Test
public void testReduceDriverMutable() {
try {
{
TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>
context =
new TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(
1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo =
(TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input =
new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(
data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
TypeComparator<Tuple2<StringValue, IntValue>> comparator =
typeInfo.createComparator(
new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());
GatheringCollector<Tuple2<StringValue, IntValue>> result =
new GatheringCollector<Tuple2<StringValue, IntValue>>(
typeInfo.createSerializer(new ExecutionConfig()));
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
context.setComparator1(comparator);
context.setCollector(result);
context.setUdf(new ConcatSumFirstMutableReducer());
ReduceCombineDriver<Tuple2<StringValue, IntValue>> driver =
new ReduceCombineDriver<Tuple2<StringValue, IntValue>>();
driver.setup(context);
driver.prepare();
driver.run();
Object[] res = result.getList().toArray();
Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
DriverTestData.compareTupleArrays(expected, res);
}
{
TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>
context =
new TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(
1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo =
(TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input =
new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(
data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
TypeComparator<Tuple2<StringValue, IntValue>> comparator =
typeInfo.createComparator(
new int[] {0}, new boolean[] {true}, 0, new ExecutionConfig());
GatheringCollector<Tuple2<StringValue, IntValue>> result =
new GatheringCollector<Tuple2<StringValue, IntValue>>(
typeInfo.createSerializer(new ExecutionConfig()));
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
context.setComparator1(comparator);
context.setCollector(result);
context.setUdf(new ConcatSumSecondMutableReducer());
ReduceCombineDriver<Tuple2<StringValue, IntValue>> driver =
new ReduceCombineDriver<Tuple2<StringValue, IntValue>>();
driver.setup(context);
driver.prepare();
driver.run();
Object[] res = result.getList().toArray();
Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
DriverTestData.compareTupleArrays(expected, res);
}
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testAllReduceDriverMutable() {
try {
{
TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>
context =
new TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TypeInformation<Tuple2<StringValue, IntValue>> typeInfo =
TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input =
new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(
data.iterator(), typeInfo.createSerializer());
GatheringCollector<Tuple2<StringValue, IntValue>> result =
new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
context.setInput1(input, typeInfo.createSerializer());
context.setCollector(result);
context.setUdf(new ConcatSumFirstMutableReducer());
AllReduceDriver<Tuple2<StringValue, IntValue>> driver =
new AllReduceDriver<Tuple2<StringValue, IntValue>>();
driver.setup(context);
driver.prepare();
driver.run();
Tuple2<StringValue, IntValue> res = result.getList().get(0);
char[] foundString = res.f0.getValue().toCharArray();
Arrays.sort(foundString);
char[] expectedString = "abcddeeeffff".toCharArray();
Arrays.sort(expectedString);
Assert.assertArrayEquals(expectedString, foundString);
Assert.assertEquals(78, res.f1.getValue());
}
{
TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>
context =
new TestTaskContext<
ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TypeInformation<Tuple2<StringValue, IntValue>> typeInfo =
TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input =
new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(
data.iterator(), typeInfo.createSerializer());
GatheringCollector<Tuple2<StringValue, IntValue>> result =
new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
context.setDriverStrategy(DriverStrategy.ALL_REDUCE);
context.setInput1(input, typeInfo.createSerializer());
context.setCollector(result);
context.setUdf(new ConcatSumSecondMutableReducer());
AllReduceDriver<Tuple2<StringValue, IntValue>> driver =
new AllReduceDriver<Tuple2<StringValue, IntValue>>();
driver.setup(context);
driver.prepare();
driver.run();
Tuple2<StringValue, IntValue> res = result.getList().get(0);
char[] foundString = res.f0.getValue().toCharArray();
Arrays.sort(foundString);
char[] expectedString = "abcddeeeffff".toCharArray();
Arrays.sort(expectedString);
Assert.assertArrayEquals(expectedString, foundString);
Assert.assertEquals(78, res.f1.getValue());
}
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}