コード例 #1
0
  @SuppressWarnings("unchecked")
  public static <K, I, O> List<StreamId> repartitionAndReduce(
      DataGraph graph,
      LocallySortedDataset<K, I> input,
      StreamReducers.Reducer<K, I, O, ?> reducer,
      List<Partition> partitions) {
    Function<I, K> keyFunction = input.keyFunction();
    List<StreamId> outputStreamIds = new ArrayList<>();
    List<NodeShard<K, I>> sharders = new ArrayList<>();
    for (StreamId inputStreamId : input.channels(graph)) {
      Partition partition = graph.getPartition(inputStreamId);
      NodeShard<K, I> sharder = new NodeShard<>(keyFunction, inputStreamId);
      graph.addNode(partition, sharder);
      sharders.add(sharder);
    }

    for (Partition partition : partitions) {
      NodeReduce<K, O, Object> streamReducer =
          new NodeReduce<>( // TODO
              input.keyComparator());
      graph.addNode(partition, streamReducer);

      for (NodeShard<K, I> sharder : sharders) {
        StreamId sharderOutput = sharder.newPartition();
        graph.addNodeStream(sharder, sharderOutput);
        StreamId reducerInput = forwardChannel(graph, input.valueType(), sharderOutput, partition);
        streamReducer.addInput(
            reducerInput, keyFunction, (StreamReducers.Reducer<K, I, O, Object>) reducer);
      }

      outputStreamIds.add(streamReducer.getOutput());
    }

    return outputStreamIds;
  }
コード例 #2
0
 private static <T> StreamId forwardChannel(
     DataGraph graph,
     Class<T> type,
     Partition sourcePartition,
     Partition targetPartition,
     StreamId sourceStreamId) {
   NodeUpload<T> nodeUpload = new NodeUpload<>(type, sourceStreamId);
   NodeDownload<T> nodeDownload =
       new NodeDownload<>(type, sourcePartition.getAddress(), sourceStreamId);
   graph.addNode(sourcePartition, nodeUpload);
   graph.addNode(targetPartition, nodeDownload);
   return nodeDownload.getOutput();
 }
コード例 #3
0
 public static <T> StreamId forwardChannel(
     DataGraph graph, Class<T> type, StreamId sourceStreamId, Partition targetPartition) {
   Partition sourcePartition = graph.getPartition(sourceStreamId);
   return forwardChannel(graph, type, sourcePartition, targetPartition, sourceStreamId);
 }