コード例 #1
0
    public void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws IOException, InterruptedException {
      int sum = 0;
      boolean filtered = true;
      String keypair = key.toString();
      String[] keys = keypair.split(" ");
      if (keys.length == 1) {
        filtered = false;
      } else {
        if (keys[0].equals("*") || keys[1].equals("*")) {
          filtered = false;
        }
      }

      if (!filtered) {
        for (IntWritable val : values) {
          sum += val.get();
        }
        context.write(key, new IntWritable(sum));
        return;
      }

      for (IntWritable val : values) {
        if (val.get() == -1) {
          filtered = false;
          continue;
        }
        sum += val.get();
      }
      // filter non-needed events
      if (filtered) return;
      context.write(key, new IntWritable(sum));
    }
コード例 #2
0
    /**
     * Map method.
     *
     * @param offset samples starting from the (offset+1)th sample.
     * @param size the number of samples for this map
     * @param context output {ture-&gt;numInside, false-&gt;numOutside}
     */
    public void map(LongWritable offset, LongWritable size, Context context)
        throws IOException, InterruptedException {

      final HaltonSequence haltonsequence = new HaltonSequence(offset.get());
      long numInside = 0L;
      long numOutside = 0L;

      for (long i = 0; i < size.get(); ) {
        // generate points in a unit square
        final double[] point = haltonsequence.nextPoint();

        // count points inside/outside of the inscribed circle of the square
        final double x = point[0] - 0.5;
        final double y = point[1] - 0.5;
        if (x * x + y * y > 0.25) {
          numOutside++;
        } else {
          numInside++;
        }

        // report status
        i++;
        if (i % 1000 == 0) {
          context.setStatus("Generated " + i + " samples.");
        }
      }

      // output map results
      context.write(new BooleanWritable(true), new LongWritable(numInside));
      context.write(new BooleanWritable(false), new LongWritable(numOutside));
    }
コード例 #3
0
    public void reduce(GFKey key, Iterable<PEIWritable> values, Context context)
        throws IOException, InterruptedException {
      // For a particular key ... process all records and output what we would have expected in this
      // concKnownKeys test
      // Note that we either
      // 1. do a single create
      // 2. create + update
      // 3. create + destroy
      // look at all ops ... and output either
      // 1. create
      // 2. create (with value from update)
      // 3. do nothing (overall result is destroy, so do not create the entry in the gemfire
      // validation region
      String keyStr = (String) key.getKey();
      ValueHolder updateValue = null;
      ValueHolder createValue = null;
      boolean destroyed = false;
      System.out.println("KnownKeysMRv2.reduce() invoked with " + keyStr);
      for (PEIWritable value : values) {
        PersistedEventImpl event = value.getEvent();
        Operation op = event.getOperation();

        ValueHolder vh = null;
        if (op.isDestroy()) {
          destroyed = true;
        } else {
          try {
            vh = (ValueHolder) event.getDeserializedValue();
          } catch (ClassNotFoundException e) {
            System.out.println(
                "KnownKeysMRv2.map() caught " + e + " : " + TestHelper.getStackTrace(e));
          }
          if (op.isUpdate()) {
            updateValue = vh;
          } else {
            createValue = vh;
          }
        }
        System.out.println(
            "KnownKeysMRv2.reduce() record: "
                + op.toString()
                + ": key = "
                + keyStr
                + " and op "
                + op.toString());
      }
      if (!destroyed) {
        if (updateValue != null) {
          context.write(key.getKey(), updateValue);
        } else {
          context.write(key.getKey(), createValue);
        }
      }
    }
コード例 #4
0
ファイル: XMLToGraph.java プロジェクト: piyushjohar/PageRank
 public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   int start = value.find("<title>");
   int end = value.find("</title>", start);
   if (start == -1 || end == -1) return;
   start += 7;
   String title = Text.decode(value.getBytes(), start, end - start);
   title = title.replace(' ', '_');
   Text titleKey = new Text(title);
   String outLinks = "";
   start = value.find("<text");
   if (start == -1) {
     context.write(titleKey, new Text(outLinks));
     return;
   }
   start = value.find(">", start);
   if (start == -1) {
     context.write(titleKey, new Text(outLinks));
     return;
   }
   end = value.find("</text>");
   if (end == -1) {
     context.write(titleKey, new Text(outLinks));
     return;
   }
   start += 1;
   String text = Text.decode(value.getBytes(), start, end - start);
   Matcher wikiLinksMatcher = patterLinks.matcher(text);
   LinkedList<String> duplicateRemover = new LinkedList<String>();
   while (wikiLinksMatcher.find()) {
     String outLinkPage = wikiLinksMatcher.group();
     outLinkPage = linksCatcher(outLinkPage);
     if (outLinkPage != null) {
       if (!outLinkPage.isEmpty()) {
         outLinkPage = outLinkPage.trim();
         duplicateRemover.add(outLinkPage);
       }
     }
   }
   LinkedHashSet<String> duplicatePruning = new LinkedHashSet<String>(duplicateRemover);
   LinkedList<String> finalList = new LinkedList<String>(duplicatePruning);
   boolean first = true;
   for (String values : finalList) {
     if (!values.equals(title)) {
       if (!first) outLinks += "\t";
       outLinks += values;
       first = false;
     }
   }
   context.write(titleKey, new Text(outLinks));
 }
  @Override
  protected void reduce(Text key, Iterable<Text> vals, Context ctx) {
    String name = null;
    String year = null;

    for (Text xx : vals) {
      String[] parts = xx.toString().split("=");

      if (parts[0].equals("name")) {
        name = parts[1];
      }

      if (parts[0].equals("year")) {
        year = parts[1];
      }
    }

    try {
      if (name != null && year != null) {
        ctx.write(new Text(year), new Text(name));
      }
    } catch (Exception ee) {
      ee.printStackTrace(System.err);
      throw new Error("I give up");
    }
  }
コード例 #6
0
ファイル: AuthorCounter.java プロジェクト: akhfa/hadoop-test
    public void map(Object key, Text value, Context context)
        throws IOException, InterruptedException {
      String file = value.toString();
      String[] lines = file.split("\n");

      for (String line : lines) {
        if (line.contains("<author>") && line.contains("</author>")) {
          String author = line.substring(8, line.indexOf("</a"));
          word.set(author);
          context.write(word, one);
        } else if (line.contains("<author>")) {
          String author = line.substring(8);
          word.set(author);
          context.write(word, one);
        }
      }
    }
コード例 #7
0
 // Identity
 public void reduce(Text key, Iterable<Text> values, Context context)
     throws IOException, InterruptedException {
   System.out.println("I'm in Job1 reduce");
   for (Text val : values) {
     System.out.println("job1:redInp:-" + val.toString());
     context.write(new Text(""), val);
   }
 }
コード例 #8
0
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
     sum += val.get();
   }
   context.write(key, new IntWritable(sum));
 }
コード例 #9
0
ファイル: X.java プロジェクト: jivesoftware/codesquare
    public void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
      System.out.println("HELLO" + key + value);

      String line = value.toString();
      String[] components = line.split("\\s+");
      context.write(new Text(components[0]), new Text(components[1]));
    }
コード例 #10
0
 public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   String line = value.toString();
   String[] currentUserTuples = line.split("::");
   int currentAgeValue = Integer.parseInt(currentUserTuples[2].trim());
   if (currentUserTuples[1].trim().equalsIgnoreCase("M") && currentAgeValue <= targetAge) {
     context.write(new Text("UserId :: " + currentUserTuples[0].trim()), one);
   }
 }
コード例 #11
0
 public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   String line = value.toString();
   StringTokenizer tokenizer = new StringTokenizer(line);
   while (tokenizer.hasMoreTokens()) {
     word.set(tokenizer.nextToken());
     context.write(word, one);
   }
 }
コード例 #12
0
ファイル: AuthorCounter.java プロジェクト: akhfa/hadoop-test
 public void reduce(Text key, Iterable<LongWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (LongWritable val : values) {
     sum += val.get();
   }
   result.set(sum);
   context.write(key, result);
 }
コード例 #13
0
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
     sum = sum + val.get();
   }
   System.out.println("----------inside-----------");
   context.write(key, new Text(Integer.toString(sum)));
 }
コード例 #14
0
 public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   Text word = new Text();
   StringTokenizer s = new StringTokenizer(value.toString());
   while (s.hasMoreTokens()) {
     word.set(s.nextToken());
     context.write(word, one);
   }
 }
コード例 #15
0
ファイル: WordMapper.java プロジェクト: xander119/hadoopEx
  public void map(LongWritable key, Text value, Context context)
      throws IOException, InterruptedException {
    String s = value.toString();

    for (String word : s.split("\\W+")) {
      if (word.length() > 0) {
        context.write(new Text(word), new IntWritable(word.length()));
      }
    }
  }
コード例 #16
0
ファイル: X.java プロジェクト: jivesoftware/codesquare
    public void reduce(Text key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {

      for (Text val : values) {
        context.write(key, val);

        System.out.println("Pass2 RED KEY: " + key);
        System.out.println("Pass2 RED VAL: " + val);
      }
    }
コード例 #17
0
ファイル: HW2_Q4.java プロジェクト: pscuderi/academic-samples
    public void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
      // read in a document (point in 58-dimensional space)

      List<Double> p = GetPoint(value.toString());

      int idxClosestCentoid = IndexOfClosestCentroid(p);

      context.write(new IntWritable(idxClosestCentoid), value);
    }
コード例 #18
0
ファイル: WordCount.java プロジェクト: patanric/Assignment2
 @Override
 public void map(Object key, Text value, Context context)
     throws IOException, InterruptedException {
   String line = value.toString();
   StringTokenizer tokenizer = new StringTokenizer(line);
   int movieId = Integer.parseInt(tokenizer.nextToken());
   while (tokenizer.hasMoreTokens()) {
     String word = tokenizer.nextToken();
     context.write(new Text("1"), new IntWritable(1));
   }
 }
コード例 #19
0
  @Override
  public void reduce(IdSetterKey key, Iterable<Text> values, Context ctx)
      throws IOException, InterruptedException {
    if (lineId == -1) {
      lineId = calculateFirstIdForReduceOutput(key);
    } else {
      lineId++;
    }

    Text outputLine = new Text(key.toString() + "#######" + values.iterator().next().toString());
    ctx.write(new LongWritable(lineId), outputLine);
  }
コード例 #20
0
    public void reduce(Text key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {
      String input[];
      double result = 0.0;

      /* adds all the values corresponding to a key */
      for (Text value : values) {
        result += Double.parseDouble(value.toString());
      }

      context.write(null, new Text(key + "," + Double.toString(result)));
    }
コード例 #21
0
ファイル: WordCount_e3.java プロジェクト: reishin/CS9223
    protected void cleanup(Context context) throws IOException, InterruptedException {

      Map<Text, IntWritable> sortedMap = sortByValues(countMap);

      int counter = 0;
      for (Text key : sortedMap.keySet()) {
        if (counter++ == 100) {
          break;
        }
        context.write(key, sortedMap.get(key));
      }
    }
コード例 #22
0
    // Identity mapper (log and write out processed key/value pairs, the value is the
    // PersistedEventImpl)
    public void map(GFKey key, PersistedEventImpl value, Context context)
        throws IOException, InterruptedException {

      String keyStr = (String) key.getKey();
      Operation op = value.getOperation();
      ValueHolder entryValue = null;
      System.out.println("map method invoked with " + keyStr + " " + op.toString());
      try {
        entryValue = (ValueHolder) value.getDeserializedValue();
      } catch (ClassNotFoundException e) {
        System.out.println("KnownKeysMRv2.map() caught " + e + " : " + TestHelper.getStackTrace(e));
      }
      context.write(key, new PEIWritable(value));
    }
コード例 #23
0
  public void reduce(LongWritable key, Iterable<Text> values, Context context)
      throws IOException, InterruptedException {
    LongWritable curNodeId = key;
    double previousPRValue = 1;
    double nextPRValue = 0;
    double localResidual = 0;
    String edgeListOfCurNode = "";
    long localResidualTransformed = 0;

    for (Text value : values) {
      String[] inputInfo = value.toString().split("\\s+");

      // incoming pagerank value
      if (inputInfo.length == 1) {
        nextPRValue += Double.parseDouble(inputInfo[0]);
      }
      // current node info
      else if (inputInfo.length == 3) {
        edgeListOfCurNode = inputInfo[2];
        previousPRValue = Double.parseDouble(inputInfo[1]);
      } else if (inputInfo.length == 2) {
        previousPRValue = Double.parseDouble(inputInfo[1]);
      } else {
        System.out.println("ERROR: received unexpected TEXT in length");
      }
    }
    if (previousPRValue == 1) System.out.println("No node info has been received by a reducer");
    // calculate the pagerank value according to the given formula
    nextPRValue = pagerankFormula(nextPRValue);

    // should also iterate sink nodes list, add the evenly splitted value

    // reducer should store the updated node info(NPR) to output directory
    context.write(null, new Text(curNodeId + " " + nextPRValue + " " + edgeListOfCurNode));

    // then compare PPR with NPR
    try {
      localResidual = Math.abs(previousPRValue - nextPRValue) / nextPRValue;
      localResidualTransformed = (long) (localResidual * 10000);
      // System.out.println("Make sure you got the right transformed residual :
      // "+localResidualTransformed);

    } catch (ArithmeticException e) {
      System.out.println("PPR is zero. Check where you get the value!");
    }

    // assume there is a global counter called residualCounter;

    context.getCounter(myCounter.ResidualCounter.RESIDUAL_SUM).increment(localResidualTransformed);
  }
コード例 #24
0
    public void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
      String line = value.toString();

      splitposition = line.indexOf("\t");
      labels = line.substring(0, splitposition);
      content = line.substring(splitposition + 1, line.length());
      if (content.length() <= 5) {
        word.set(labels);
        int counter = Integer.parseInt(content);
        context.write(word, new IntWritable(counter));
        return;
      }
      labeltokens = labels.split(",");
      contenttokens = tokenizeDoc(content);

      for (String label : labelspace) {
        for (String token : contenttokens) {
          // filter token, denotes the needed events;
          word.set(label + " " + token);
          context.write(word, new IntWritable(-1));
        }
      }
    }
コード例 #25
0
ファイル: HW2_Q4.java プロジェクト: pscuderi/academic-samples
    public void reduce(IntWritable key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {
      List<Double> newCentroid = null;
      int numPoints = 0;

      for (Text value : values) {
        ++numPoints;

        List<Double> p = GetPoint(value.toString());

        if (newCentroid == null) {
          // initialize the new centroid to the first element
          newCentroid = new ArrayList<Double>(p);
        } else {
          for (int i = 0; i < newCentroid.size(); i++) {
            newCentroid.set(i, newCentroid.get(i) + p.get(i));
          }
        }
      }

      // now the newCentroid contains the sum of all the points
      // so to get the average of all the points, we need to
      // divide each entry by the total number of points

      for (int i = 0; i < newCentroid.size(); i++) {
        newCentroid.set(i, newCentroid.get(i) / (double) numPoints);
      }

      // now create a string containing all the new centroid's coordinates

      String s = null;
      for (Double d : newCentroid) {
        if (s == null) {
          s = d.toString();
        } else {
          s += " " + d.toString();
        }
      }

      newCentroids.add(s);

      if (newCentroids.size() == 10) {
        WriteNewCentroids(context);
      }

      // output the centroid ID and the centroid data
      context.write(key, new Text(s));
    }
コード例 #26
0
 public void map(LongWritable key, Text value, Context context)
     throws IOException, InterruptedException {
   String line = value.toString();
   String[] currentMovieTuples = line.split("::");
   String[] inMovieList = inMovies.split(",");
   for (String s : inMovieList) {
     if (currentMovieTuples[1].trim().equalsIgnoreCase(s))
       context.write(
           new Text(
               "Movie :: "
                   + currentMovieTuples[1].trim()
                   + " Genre :: "
                   + currentMovieTuples[2].trim()),
           one);
   }
 }
コード例 #27
0
    public void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
      Configuration c = context.getConfiguration();
      String s = value.toString();
      String input[] = s.split(",");
      Text outputkey = new Text();
      Text outputvalue = new Text();
      double result = 0.0;

      /* multiplies matrix and vector entry with matching column value */

      result = (Double.parseDouble(input[2])) * (vector.get(Long.parseLong(input[1])));
      outputkey.set(input[0]);
      outputvalue.set(Double.toString(result));
      context.write(outputkey, outputvalue);
    }
コード例 #28
0
ファイル: KmeansMapper.java プロジェクト: whiker/Kmeans
  @Override
  public void map(LongWritable key, Text value, Context context)
      throws IOException, InterruptedException {
    String str = "1\t" + value.toString();
    Kvector vector = Kvector.fromString(str);

    int code = -1, d = Integer.MAX_VALUE;
    for (Kvector e : m_kVectors) {
      int t = vector.distance(e);
      if (t < d) {
        d = t;
        code = e.code;
      }
    }

    context.write(new KcodeWritable(code), new Text(str));
  }
コード例 #29
0
 public void reduce(StockRecord key, Iterable<NullWritable> values, Context context)
     throws IOException, InterruptedException {
   context.write(key, NullWritable.get());
 }
コード例 #30
0
 @Override
 protected void map(LongWritable key, StockRecord value, Context context)
     throws IOException, InterruptedException {
   context.write(value, NullWritable.get());
 }