示例#1
0
  public static Fields getOutputFields(List<CombinerDefinition> combinerDefinitions) {
    Fields summedOutputFields = new Fields(MultiCombiner.ID_FIELD);

    for (CombinerDefinition combinerDefinition : combinerDefinitions) {
      summedOutputFields =
          Fields.merge(
              summedOutputFields,
              combinerDefinition.getGroupFields(),
              combinerDefinition.getOutputFields());
    }
    return summedOutputFields;
  }
示例#2
0
  public static <T> void populateOutputTupleEntry(
      CombinerDefinition<T> definition, TupleEntry output, Tuple resultTuple) {
    // set the ID so we can differentiate later
    output.setRaw(MultiCombiner.ID_FIELD, definition.getId());

    // our tuples are of the form groupFields+outputFields, set the TupleEntry fields appropriately
    Fields groupFields = definition.getGroupFields();
    int index = 0;
    for (int i = 0; i < groupFields.size(); i++) {
      output.setRaw(groupFields.get(i), resultTuple.getObject(index));
      index++;
    }
    Fields outputFields = definition.getOutputFields();
    for (int i = 0; i < outputFields.size(); i++) {
      output.setRaw(outputFields.get(i), resultTuple.getObject(index));
      index++;
    }
  }
示例#3
0
  public MultiCombiner(
      Pipe[] pipes, List<CombinerDefinition> combinerDefinitions, boolean filterTails) {
    super(pipes);

    if (combinerIdsCollide(combinerDefinitions)) {
      throw new IllegalArgumentException(
          "Some CombinerDefinition ids collide. "
              + "Make sure that all names are unique and, if they are, check for hashCode collisions");
    }

    Pipe[] pipesCopy = new Pipe[pipes.length];
    for (int i = 0; i < pipes.length; i++) {
      pipesCopy[i] =
          new Each(pipes[i], new MultiCombinerFunction(combinerDefinitions), Fields.RESULTS);
    }
    Pipe pipe = new GroupBy(pipesCopy, determineGroupFields(combinerDefinitions));
    pipe = new Every(pipe, new MultiCombinerAggregator(combinerDefinitions), Fields.RESULTS);

    if (filterTails) {
      Pipe[] tails = new Pipe[combinerDefinitions.size()];

      for (int i = 0; i < combinerDefinitions.size(); i++) {
        CombinerDefinition definition = combinerDefinitions.get(i);
        Pipe output = new Pipe(definition.getName() + "-intermediate", pipe);
        output =
            new Each(output, new Fields(ID_FIELD), new MultiCombinerFilter(definition.getId()));
        output =
            new Retain(output, definition.getGroupFields().append(definition.getOutputFields()));
        tails[i] = new Pipe(definition.getName(), output);
      }

      setTails(tails);
    } else {
      setTails(pipe);
    }
  }