public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; boolean filtered = true; String keypair = key.toString(); String[] keys = keypair.split(" "); if (keys.length == 1) { filtered = false; } else { if (keys[0].equals("*") || keys[1].equals("*")) { filtered = false; } } if (!filtered) { for (IntWritable val : values) { sum += val.get(); } context.write(key, new IntWritable(sum)); return; } for (IntWritable val : values) { if (val.get() == -1) { filtered = false; continue; } sum += val.get(); } // filter non-needed events if (filtered) return; context.write(key, new IntWritable(sum)); }
/** * Map method. * * @param offset samples starting from the (offset+1)th sample. * @param size the number of samples for this map * @param context output {ture->numInside, false->numOutside} */ public void map(LongWritable offset, LongWritable size, Context context) throws IOException, InterruptedException { final HaltonSequence haltonsequence = new HaltonSequence(offset.get()); long numInside = 0L; long numOutside = 0L; for (long i = 0; i < size.get(); ) { // generate points in a unit square final double[] point = haltonsequence.nextPoint(); // count points inside/outside of the inscribed circle of the square final double x = point[0] - 0.5; final double y = point[1] - 0.5; if (x * x + y * y > 0.25) { numOutside++; } else { numInside++; } // report status i++; if (i % 1000 == 0) { context.setStatus("Generated " + i + " samples."); } } // output map results context.write(new BooleanWritable(true), new LongWritable(numInside)); context.write(new BooleanWritable(false), new LongWritable(numOutside)); }
public void reduce(GFKey key, Iterable<PEIWritable> values, Context context) throws IOException, InterruptedException { // For a particular key ... process all records and output what we would have expected in this // concKnownKeys test // Note that we either // 1. do a single create // 2. create + update // 3. create + destroy // look at all ops ... and output either // 1. create // 2. create (with value from update) // 3. do nothing (overall result is destroy, so do not create the entry in the gemfire // validation region String keyStr = (String) key.getKey(); ValueHolder updateValue = null; ValueHolder createValue = null; boolean destroyed = false; System.out.println("KnownKeysMRv2.reduce() invoked with " + keyStr); for (PEIWritable value : values) { PersistedEventImpl event = value.getEvent(); Operation op = event.getOperation(); ValueHolder vh = null; if (op.isDestroy()) { destroyed = true; } else { try { vh = (ValueHolder) event.getDeserializedValue(); } catch (ClassNotFoundException e) { System.out.println( "KnownKeysMRv2.map() caught " + e + " : " + TestHelper.getStackTrace(e)); } if (op.isUpdate()) { updateValue = vh; } else { createValue = vh; } } System.out.println( "KnownKeysMRv2.reduce() record: " + op.toString() + ": key = " + keyStr + " and op " + op.toString()); } if (!destroyed) { if (updateValue != null) { context.write(key.getKey(), updateValue); } else { context.write(key.getKey(), createValue); } } }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { int start = value.find("<title>"); int end = value.find("</title>", start); if (start == -1 || end == -1) return; start += 7; String title = Text.decode(value.getBytes(), start, end - start); title = title.replace(' ', '_'); Text titleKey = new Text(title); String outLinks = ""; start = value.find("<text"); if (start == -1) { context.write(titleKey, new Text(outLinks)); return; } start = value.find(">", start); if (start == -1) { context.write(titleKey, new Text(outLinks)); return; } end = value.find("</text>"); if (end == -1) { context.write(titleKey, new Text(outLinks)); return; } start += 1; String text = Text.decode(value.getBytes(), start, end - start); Matcher wikiLinksMatcher = patterLinks.matcher(text); LinkedList<String> duplicateRemover = new LinkedList<String>(); while (wikiLinksMatcher.find()) { String outLinkPage = wikiLinksMatcher.group(); outLinkPage = linksCatcher(outLinkPage); if (outLinkPage != null) { if (!outLinkPage.isEmpty()) { outLinkPage = outLinkPage.trim(); duplicateRemover.add(outLinkPage); } } } LinkedHashSet<String> duplicatePruning = new LinkedHashSet<String>(duplicateRemover); LinkedList<String> finalList = new LinkedList<String>(duplicatePruning); boolean first = true; for (String values : finalList) { if (!values.equals(title)) { if (!first) outLinks += "\t"; outLinks += values; first = false; } } context.write(titleKey, new Text(outLinks)); }
@Override protected void reduce(Text key, Iterable<Text> vals, Context ctx) { String name = null; String year = null; for (Text xx : vals) { String[] parts = xx.toString().split("="); if (parts[0].equals("name")) { name = parts[1]; } if (parts[0].equals("year")) { year = parts[1]; } } try { if (name != null && year != null) { ctx.write(new Text(year), new Text(name)); } } catch (Exception ee) { ee.printStackTrace(System.err); throw new Error("I give up"); } }
public void map(Object key, Text value, Context context) throws IOException, InterruptedException { String file = value.toString(); String[] lines = file.split("\n"); for (String line : lines) { if (line.contains("<author>") && line.contains("</author>")) { String author = line.substring(8, line.indexOf("</a")); word.set(author); context.write(word, one); } else if (line.contains("<author>")) { String author = line.substring(8); word.set(author); context.write(word, one); } } }
// Identity public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { System.out.println("I'm in Job1 reduce"); for (Text val : values) { System.out.println("job1:redInp:-" + val.toString()); context.write(new Text(""), val); } }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } context.write(key, new IntWritable(sum)); }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { System.out.println("HELLO" + key + value); String line = value.toString(); String[] components = line.split("\\s+"); context.write(new Text(components[0]), new Text(components[1])); }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); String[] currentUserTuples = line.split("::"); int currentAgeValue = Integer.parseInt(currentUserTuples[2].trim()); if (currentUserTuples[1].trim().equalsIgnoreCase("M") && currentAgeValue <= targetAge) { context.write(new Text("UserId :: " + currentUserTuples[0].trim()), one); } }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); StringTokenizer tokenizer = new StringTokenizer(line); while (tokenizer.hasMoreTokens()) { word.set(tokenizer.nextToken()); context.write(word, one); } }
public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (LongWritable val : values) { sum += val.get(); } result.set(sum); context.write(key, result); }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum = sum + val.get(); } System.out.println("----------inside-----------"); context.write(key, new Text(Integer.toString(sum))); }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { Text word = new Text(); StringTokenizer s = new StringTokenizer(value.toString()); while (s.hasMoreTokens()) { word.set(s.nextToken()); context.write(word, one); } }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String s = value.toString(); for (String word : s.split("\\W+")) { if (word.length() > 0) { context.write(new Text(word), new IntWritable(word.length())); } } }
public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { for (Text val : values) { context.write(key, val); System.out.println("Pass2 RED KEY: " + key); System.out.println("Pass2 RED VAL: " + val); } }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { // read in a document (point in 58-dimensional space) List<Double> p = GetPoint(value.toString()); int idxClosestCentoid = IndexOfClosestCentroid(p); context.write(new IntWritable(idxClosestCentoid), value); }
@Override public void map(Object key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); StringTokenizer tokenizer = new StringTokenizer(line); int movieId = Integer.parseInt(tokenizer.nextToken()); while (tokenizer.hasMoreTokens()) { String word = tokenizer.nextToken(); context.write(new Text("1"), new IntWritable(1)); } }
@Override public void reduce(IdSetterKey key, Iterable<Text> values, Context ctx) throws IOException, InterruptedException { if (lineId == -1) { lineId = calculateFirstIdForReduceOutput(key); } else { lineId++; } Text outputLine = new Text(key.toString() + "#######" + values.iterator().next().toString()); ctx.write(new LongWritable(lineId), outputLine); }
public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { String input[]; double result = 0.0; /* adds all the values corresponding to a key */ for (Text value : values) { result += Double.parseDouble(value.toString()); } context.write(null, new Text(key + "," + Double.toString(result))); }
protected void cleanup(Context context) throws IOException, InterruptedException { Map<Text, IntWritable> sortedMap = sortByValues(countMap); int counter = 0; for (Text key : sortedMap.keySet()) { if (counter++ == 100) { break; } context.write(key, sortedMap.get(key)); } }
// Identity mapper (log and write out processed key/value pairs, the value is the // PersistedEventImpl) public void map(GFKey key, PersistedEventImpl value, Context context) throws IOException, InterruptedException { String keyStr = (String) key.getKey(); Operation op = value.getOperation(); ValueHolder entryValue = null; System.out.println("map method invoked with " + keyStr + " " + op.toString()); try { entryValue = (ValueHolder) value.getDeserializedValue(); } catch (ClassNotFoundException e) { System.out.println("KnownKeysMRv2.map() caught " + e + " : " + TestHelper.getStackTrace(e)); } context.write(key, new PEIWritable(value)); }
public void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { LongWritable curNodeId = key; double previousPRValue = 1; double nextPRValue = 0; double localResidual = 0; String edgeListOfCurNode = ""; long localResidualTransformed = 0; for (Text value : values) { String[] inputInfo = value.toString().split("\\s+"); // incoming pagerank value if (inputInfo.length == 1) { nextPRValue += Double.parseDouble(inputInfo[0]); } // current node info else if (inputInfo.length == 3) { edgeListOfCurNode = inputInfo[2]; previousPRValue = Double.parseDouble(inputInfo[1]); } else if (inputInfo.length == 2) { previousPRValue = Double.parseDouble(inputInfo[1]); } else { System.out.println("ERROR: received unexpected TEXT in length"); } } if (previousPRValue == 1) System.out.println("No node info has been received by a reducer"); // calculate the pagerank value according to the given formula nextPRValue = pagerankFormula(nextPRValue); // should also iterate sink nodes list, add the evenly splitted value // reducer should store the updated node info(NPR) to output directory context.write(null, new Text(curNodeId + " " + nextPRValue + " " + edgeListOfCurNode)); // then compare PPR with NPR try { localResidual = Math.abs(previousPRValue - nextPRValue) / nextPRValue; localResidualTransformed = (long) (localResidual * 10000); // System.out.println("Make sure you got the right transformed residual : // "+localResidualTransformed); } catch (ArithmeticException e) { System.out.println("PPR is zero. Check where you get the value!"); } // assume there is a global counter called residualCounter; context.getCounter(myCounter.ResidualCounter.RESIDUAL_SUM).increment(localResidualTransformed); }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); splitposition = line.indexOf("\t"); labels = line.substring(0, splitposition); content = line.substring(splitposition + 1, line.length()); if (content.length() <= 5) { word.set(labels); int counter = Integer.parseInt(content); context.write(word, new IntWritable(counter)); return; } labeltokens = labels.split(","); contenttokens = tokenizeDoc(content); for (String label : labelspace) { for (String token : contenttokens) { // filter token, denotes the needed events; word.set(label + " " + token); context.write(word, new IntWritable(-1)); } } }
public void reduce(IntWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { List<Double> newCentroid = null; int numPoints = 0; for (Text value : values) { ++numPoints; List<Double> p = GetPoint(value.toString()); if (newCentroid == null) { // initialize the new centroid to the first element newCentroid = new ArrayList<Double>(p); } else { for (int i = 0; i < newCentroid.size(); i++) { newCentroid.set(i, newCentroid.get(i) + p.get(i)); } } } // now the newCentroid contains the sum of all the points // so to get the average of all the points, we need to // divide each entry by the total number of points for (int i = 0; i < newCentroid.size(); i++) { newCentroid.set(i, newCentroid.get(i) / (double) numPoints); } // now create a string containing all the new centroid's coordinates String s = null; for (Double d : newCentroid) { if (s == null) { s = d.toString(); } else { s += " " + d.toString(); } } newCentroids.add(s); if (newCentroids.size() == 10) { WriteNewCentroids(context); } // output the centroid ID and the centroid data context.write(key, new Text(s)); }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); String[] currentMovieTuples = line.split("::"); String[] inMovieList = inMovies.split(","); for (String s : inMovieList) { if (currentMovieTuples[1].trim().equalsIgnoreCase(s)) context.write( new Text( "Movie :: " + currentMovieTuples[1].trim() + " Genre :: " + currentMovieTuples[2].trim()), one); } }
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { Configuration c = context.getConfiguration(); String s = value.toString(); String input[] = s.split(","); Text outputkey = new Text(); Text outputvalue = new Text(); double result = 0.0; /* multiplies matrix and vector entry with matching column value */ result = (Double.parseDouble(input[2])) * (vector.get(Long.parseLong(input[1]))); outputkey.set(input[0]); outputvalue.set(Double.toString(result)); context.write(outputkey, outputvalue); }
@Override public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String str = "1\t" + value.toString(); Kvector vector = Kvector.fromString(str); int code = -1, d = Integer.MAX_VALUE; for (Kvector e : m_kVectors) { int t = vector.distance(e); if (t < d) { d = t; code = e.code; } } context.write(new KcodeWritable(code), new Text(str)); }
public void reduce(StockRecord key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException { context.write(key, NullWritable.get()); }
@Override protected void map(LongWritable key, StockRecord value, Context context) throws IOException, InterruptedException { context.write(value, NullWritable.get()); }