Beispiel #1
0
 @Override
 protected void reduce(Centroid key, Iterable<Point> values, Context context)
     throws IOException, InterruptedException {
   ArrayList<Point> points = new ArrayList<Point>();
   points.clear();
   int clusterId = key.id;
   Point newCenter = null;
   for (Point p : values) {
     Point copy = p.deepCopy();
     points.add(copy);
     if (newCenter == null) {
       newCenter = new Point(copy.deepCopy());
     } else {
       newCenter = newCenter.add(copy);
     }
   }
   Centroid center = new Centroid(clusterId, newCenter);
   for (Point p : points) {
     context.write(center, p);
   }
 }
  protected void reduce(Text key, Iterable<Text> values, Context context)
      throws IOException, InterruptedException {

    Iterator<Text> itr = values.iterator();
    Text input = new Text();
    String[] inputTokens = null;

    // initialize/reset all variables
    Double pageRankOld = 0.0;
    Double residualError = 0.0;

    String output = "";
    Integer maxNode = 0;

    ArrayList<String> temp = new ArrayList<String>();
    Double tempBC = 0.0;
    vList.clear();
    newPR.clear();
    BE.clear();
    BC.clear();
    nodeDataMap.clear();

    while (itr.hasNext()) {
      input = itr.next();
      inputTokens = input.toString().split(" ");
      // if first element is PR, it is the node ID, previous pagerank and outgoing edgelist for this
      // node
      if (inputTokens[0].equals("PR")) {
        String nodeID = inputTokens[1];
        pageRankOld = Double.parseDouble(inputTokens[2]);
        newPR.put(nodeID, pageRankOld);
        NodeData node = new NodeData();
        node.setNodeID(nodeID);
        node.setPageRank(pageRankOld);
        if (inputTokens.length == 4) {
          node.setEdgeList(inputTokens[3]);
          node.setDegrees(inputTokens[3].split(",").length);
        }
        vList.add(nodeID);
        nodeDataMap.put(nodeID, node);
        // keep track of the max nodeID for this block
        if (Integer.parseInt(nodeID) > maxNode) {
          maxNode = Integer.parseInt(nodeID);
        }

        // if BE, it is an in-block edge
      } else if (inputTokens[0].equals("BE")) {

        if (BE.containsKey(inputTokens[2])) {
          // Initialize BC for this v
          temp = BE.get(inputTokens[2]);
        } else {
          temp = new ArrayList<String>();
        }
        temp.add(inputTokens[1]);
        BE.put(inputTokens[2], temp);

        // if BC, it is an incoming node from outside of the block
      } else if (inputTokens[0].equals("BC")) {
        if (BC.containsKey(inputTokens[2])) {
          // Initialize BC for this v
          tempBC = BC.get(inputTokens[2]);
        } else {
          tempBC = 0.0;
        }
        tempBC += Double.parseDouble(inputTokens[3]);
        BC.put(inputTokens[2], tempBC);
      }
    }

    int i = 0;
    do {
      i++;
      residualError = IterateBlockOnce();
      // System.out.println("Block " + key + " pass " + i + " resError:" + residualError);
    } while (residualError > threshold);

    // i < maxIterations &&

    // compute the ultimate residual error for each node in this block
    residualError = 0.0;
    for (String v : vList) {
      NodeData node = nodeDataMap.get(v);
      residualError += Math.abs(node.getPageRank() - newPR.get(v)) / newPR.get(v);
    }
    residualError = residualError / vList.size();
    // System.out.println("Block " + key + " overall resError for iteration: " + residualError);

    // add the residual error to the counter that is tracking the overall sum (must be expressed as
    // a long value)
    long residualAsLong = (long) Math.floor(residualError * PageRankBlock.precision);
    long numberOfIterations = (long) (i);
    context.getCounter(PageRankBlock.ProjectCounters.RESIDUAL_ERROR).increment(residualAsLong);

    context
        .getCounter(PageRankBlock.ProjectCounters.AVERAGE_ITERATIONS)
        .increment(numberOfIterations);

    // output should be
    //	key:nodeID (for this node)
    //	value:<pageRankNew> <degrees> <comma-separated outgoing edgeList>
    for (String v : vList) {
      NodeData node = nodeDataMap.get(v);
      output = newPR.get(v) + " " + node.getDegrees() + " " + node.getEdgeList();
      Text outputText = new Text(output);
      Text outputKey = new Text(v);
      context.write(outputKey, outputText);
      if (v.equals(maxNode.toString())) {
        System.out.println("Block:" + key + " | node:" + v + " | pageRank:" + newPR.get(v));
      }
    }

    cleanup(context);
  }