private static Fields determineGroupFields(List<CombinerDefinition> combinerDefinitions) { Fields summedGroupFields = new Fields(MultiCombiner.ID_FIELD); for (CombinerDefinition def : combinerDefinitions) { summedGroupFields = Fields.merge(summedGroupFields, def.getGroupFields()); } return summedGroupFields; }
public static Fields getIntermediateFields(List<CombinerDefinition> combinerDefinitions) { Fields summedIntermediateFields = new Fields(MultiCombiner.ID_FIELD); for (CombinerDefinition combinerDefinition : combinerDefinitions) { summedIntermediateFields = Fields.merge( summedIntermediateFields, combinerDefinition.getGroupFields(), combinerDefinition.getIntermediateFields()); } return summedIntermediateFields; }
public static Fields getInputFields(List<CombinerDefinition> combinerDefinitions) { Fields summedInputFields = new Fields(); for (CombinerDefinition combinerDefinition : combinerDefinitions) { summedInputFields = Fields.merge( summedInputFields, combinerDefinition.getGroupFields(), combinerDefinition.getInputFields()); } return summedInputFields; }
public static <T> void populateOutputTupleEntry( CombinerDefinition<T> definition, TupleEntry output, Tuple resultTuple) { // set the ID so we can differentiate later output.setRaw(MultiCombiner.ID_FIELD, definition.getId()); // our tuples are of the form groupFields+outputFields, set the TupleEntry fields appropriately Fields groupFields = definition.getGroupFields(); int index = 0; for (int i = 0; i < groupFields.size(); i++) { output.setRaw(groupFields.get(i), resultTuple.getObject(index)); index++; } Fields outputFields = definition.getOutputFields(); for (int i = 0; i < outputFields.size(); i++) { output.setRaw(outputFields.get(i), resultTuple.getObject(index)); index++; } }
public MultiCombiner( Pipe[] pipes, List<CombinerDefinition> combinerDefinitions, boolean filterTails) { super(pipes); if (combinerIdsCollide(combinerDefinitions)) { throw new IllegalArgumentException( "Some CombinerDefinition ids collide. " + "Make sure that all names are unique and, if they are, check for hashCode collisions"); } Pipe[] pipesCopy = new Pipe[pipes.length]; for (int i = 0; i < pipes.length; i++) { pipesCopy[i] = new Each(pipes[i], new MultiCombinerFunction(combinerDefinitions), Fields.RESULTS); } Pipe pipe = new GroupBy(pipesCopy, determineGroupFields(combinerDefinitions)); pipe = new Every(pipe, new MultiCombinerAggregator(combinerDefinitions), Fields.RESULTS); if (filterTails) { Pipe[] tails = new Pipe[combinerDefinitions.size()]; for (int i = 0; i < combinerDefinitions.size(); i++) { CombinerDefinition definition = combinerDefinitions.get(i); Pipe output = new Pipe(definition.getName() + "-intermediate", pipe); output = new Each(output, new Fields(ID_FIELD), new MultiCombinerFilter(definition.getId())); output = new Retain(output, definition.getGroupFields().append(definition.getOutputFields())); tails[i] = new Pipe(definition.getName(), output); } setTails(tails); } else { setTails(pipe); } }