/** * Group by and combine adding all values from table (1,2),(1,3),(2,4),(2,6),(3,6),(3,9) Should * raise result (1,(2+3)),(2,(4+6)),(3,(6+9)) = (1,5),(2,10),(3,15) */ @Test public void testCombine() { DoFn<Integer, Pair<Integer, Integer>> fn = new DoFn<Integer, Pair<Integer, Integer>>() { @Override public void process(Integer v, EmitFn<Pair<Integer, Integer>> emitter) { emitter.emit(Pair.create(v, v * 2)); emitter.emit(Pair.create(v, v * 3)); } }; // Get Plume runtime LazyPlume plume = new LazyPlume(); List<Integer> l1 = Lists.newArrayList(1, 2, 3); PTable<Integer, Integer> output = plume .fromJava(l1) .map(fn, tableOf(integers(), integers())) .groupByKey() .combine( new CombinerFn<Integer>() { @Override public Integer combine(Iterable<Integer> stuff) { Integer result = 0; for (Integer i : stuff) { result += i; } return result; } }); // Get an executor LocalExecutor executor = new LocalExecutor(); LazyTable<Integer, Integer> lOutput = (LazyTable<Integer, Integer>) output; Iterable<Pair<Integer, Integer>> executedOutput = executor.execute(lOutput); List<Pair<Integer, Integer>> outputList = Lists.newArrayList(executedOutput); Collections.sort( outputList, new Comparator<Pair<Integer, ?>>() { @Override public int compare(Pair<Integer, ?> arg0, Pair<Integer, ?> arg1) { return arg0.getKey().compareTo(arg1.getKey()); } }); assertEquals(outputList.get(0).getKey().intValue(), 1); assertEquals(outputList.get(0).getValue().intValue(), 5); assertEquals(outputList.get(1).getKey().intValue(), 2); assertEquals(outputList.get(1).getValue().intValue(), 10); assertEquals(outputList.get(2).getKey().intValue(), 3); assertEquals(outputList.get(2).getValue().intValue(), 15); }
/** Try one group by from table (1,2),(1,3),(2,4),(2,6),(3,6),(3,9) */ @Test public void testGroupByKey() { // Get Plume runtime LazyPlume plume = new LazyPlume(); List<Integer> l1 = Lists.newArrayList(1, 2, 3); PTable<Integer, Iterable<Integer>> output = plume.fromJava(l1).map(plusTwoPlusThree, tableOf(integers(), integers())).groupByKey(); LazyTable<Integer, Iterable<Integer>> lOutput = (LazyTable<Integer, Iterable<Integer>>) output; // Get an executor LocalExecutor executor = new LocalExecutor(); Iterable<Pair<Integer, Iterable<Integer>>> executedOutput = executor.execute(lOutput); List<Pair<Integer, Iterable<Integer>>> outputList = Lists.newArrayList(executedOutput); Collections.sort( outputList, new Comparator<Pair<Integer, ?>>() { @Override public int compare(Pair<Integer, ?> arg0, Pair<Integer, ?> arg1) { return arg0.getKey().compareTo(arg1.getKey()); } }); assertEquals(outputList.get(0).getKey().intValue(), 1); assertEquals(outputList.get(1).getKey().intValue(), 2); assertEquals(outputList.get(2).getKey().intValue(), 3); List<Integer> lR1 = Lists.newArrayList(outputList.get(0).getValue()); List<Integer> lR2 = Lists.newArrayList(outputList.get(1).getValue()); List<Integer> lR3 = Lists.newArrayList(outputList.get(2).getValue()); Collections.sort(lR1); Collections.sort(lR2); Collections.sort(lR3); assertEquals(lR1.get(0).intValue(), 2); assertEquals(lR1.get(1).intValue(), 3); assertEquals(lR2.get(0).intValue(), 4); assertEquals(lR2.get(1).intValue(), 6); assertEquals(lR3.get(0).intValue(), 6); assertEquals(lR3.get(1).intValue(), 9); }