public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
boolean filtered = true;
String keypair = key.toString();
String[] keys = keypair.split(" ");
if (keys.length == 1) {
filtered = false;
} else {
if (keys[0].equals("*") || keys[1].equals("*")) {
filtered = false;
}
}
if (!filtered) {
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
return;
}
for (IntWritable val : values) {
if (val.get() == -1) {
filtered = false;
continue;
}
sum += val.get();
}
// filter non-needed events
if (filtered) return;
context.write(key, new IntWritable(sum));
}
@Override
public void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
Cluster cluster = new Cluster(conf);
Job currentJob = cluster.getJob(context.getJobID());
counterGroup = currentJob.getCounters();
}
/**
* Map method.
*
* @param offset samples starting from the (offset+1)th sample.
* @param size the number of samples for this map
* @param context output {ture->numInside, false->numOutside}
*/
public void map(LongWritable offset, LongWritable size, Context context)
throws IOException, InterruptedException {
final HaltonSequence haltonsequence = new HaltonSequence(offset.get());
long numInside = 0L;
long numOutside = 0L;
for (long i = 0; i < size.get(); ) {
// generate points in a unit square
final double[] point = haltonsequence.nextPoint();
// count points inside/outside of the inscribed circle of the square
final double x = point[0] - 0.5;
final double y = point[1] - 0.5;
if (x * x + y * y > 0.25) {
numOutside++;
} else {
numInside++;
}
// report status
i++;
if (i % 1000 == 0) {
context.setStatus("Generated " + i + " samples.");
}
}
// output map results
context.write(new BooleanWritable(true), new LongWritable(numInside));
context.write(new BooleanWritable(false), new LongWritable(numOutside));
}
public void reduce(GFKey key, Iterable<PEIWritable> values, Context context)
throws IOException, InterruptedException {
// For a particular key ... process all records and output what we would have expected in this
// concKnownKeys test
// Note that we either
// 1. do a single create
// 2. create + update
// 3. create + destroy
// look at all ops ... and output either
// 1. create
// 2. create (with value from update)
// 3. do nothing (overall result is destroy, so do not create the entry in the gemfire
// validation region
String keyStr = (String) key.getKey();
ValueHolder updateValue = null;
ValueHolder createValue = null;
boolean destroyed = false;
System.out.println("KnownKeysMRv2.reduce() invoked with " + keyStr);
for (PEIWritable value : values) {
PersistedEventImpl event = value.getEvent();
Operation op = event.getOperation();
ValueHolder vh = null;
if (op.isDestroy()) {
destroyed = true;
} else {
try {
vh = (ValueHolder) event.getDeserializedValue();
} catch (ClassNotFoundException e) {
System.out.println(
"KnownKeysMRv2.map() caught " + e + " : " + TestHelper.getStackTrace(e));
}
if (op.isUpdate()) {
updateValue = vh;
} else {
createValue = vh;
}
}
System.out.println(
"KnownKeysMRv2.reduce() record: "
+ op.toString()
+ ": key = "
+ keyStr
+ " and op "
+ op.toString());
}
if (!destroyed) {
if (updateValue != null) {
context.write(key.getKey(), updateValue);
} else {
context.write(key.getKey(), createValue);
}
}
}
private static void WriteNewCentroids(Context context) throws IOException {
FileSystem fs = FileSystem.get(context.getConfiguration());
Path nextCFile = new Path(context.getConfiguration().get("NEXTCFILE"));
DataOutputStream d = new DataOutputStream(fs.create(nextCFile, false));
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(d));
for (String centroid : newCentroids) {
writer.write(centroid + "\n");
}
writer.close();
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
int start = value.find("<title>");
int end = value.find("</title>", start);
if (start == -1 || end == -1) return;
start += 7;
String title = Text.decode(value.getBytes(), start, end - start);
title = title.replace(' ', '_');
Text titleKey = new Text(title);
String outLinks = "";
start = value.find("<text");
if (start == -1) {
context.write(titleKey, new Text(outLinks));
return;
}
start = value.find(">", start);
if (start == -1) {
context.write(titleKey, new Text(outLinks));
return;
}
end = value.find("</text>");
if (end == -1) {
context.write(titleKey, new Text(outLinks));
return;
}
start += 1;
String text = Text.decode(value.getBytes(), start, end - start);
Matcher wikiLinksMatcher = patterLinks.matcher(text);
LinkedList<String> duplicateRemover = new LinkedList<String>();
while (wikiLinksMatcher.find()) {
String outLinkPage = wikiLinksMatcher.group();
outLinkPage = linksCatcher(outLinkPage);
if (outLinkPage != null) {
if (!outLinkPage.isEmpty()) {
outLinkPage = outLinkPage.trim();
duplicateRemover.add(outLinkPage);
}
}
}
LinkedHashSet<String> duplicatePruning = new LinkedHashSet<String>(duplicateRemover);
LinkedList<String> finalList = new LinkedList<String>(duplicatePruning);
boolean first = true;
for (String values : finalList) {
if (!values.equals(title)) {
if (!first) outLinks += "\t";
outLinks += values;
first = false;
}
}
context.write(titleKey, new Text(outLinks));
}
public void reduce(LongWritable key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
LongWritable curNodeId = key;
double previousPRValue = 1;
double nextPRValue = 0;
double localResidual = 0;
String edgeListOfCurNode = "";
long localResidualTransformed = 0;
for (Text value : values) {
String[] inputInfo = value.toString().split("\\s+");
// incoming pagerank value
if (inputInfo.length == 1) {
nextPRValue += Double.parseDouble(inputInfo[0]);
}
// current node info
else if (inputInfo.length == 3) {
edgeListOfCurNode = inputInfo[2];
previousPRValue = Double.parseDouble(inputInfo[1]);
} else if (inputInfo.length == 2) {
previousPRValue = Double.parseDouble(inputInfo[1]);
} else {
System.out.println("ERROR: received unexpected TEXT in length");
}
}
if (previousPRValue == 1) System.out.println("No node info has been received by a reducer");
// calculate the pagerank value according to the given formula
nextPRValue = pagerankFormula(nextPRValue);
// should also iterate sink nodes list, add the evenly splitted value
// reducer should store the updated node info(NPR) to output directory
context.write(null, new Text(curNodeId + " " + nextPRValue + " " + edgeListOfCurNode));
// then compare PPR with NPR
try {
localResidual = Math.abs(previousPRValue - nextPRValue) / nextPRValue;
localResidualTransformed = (long) (localResidual * 10000);
// System.out.println("Make sure you got the right transformed residual :
// "+localResidualTransformed);
} catch (ArithmeticException e) {
System.out.println("PPR is zero. Check where you get the value!");
}
// assume there is a global counter called residualCounter;
context.getCounter(myCounter.ResidualCounter.RESIDUAL_SUM).increment(localResidualTransformed);
}
@Override
protected void reduce(Text key, Iterable<Text> vals, Context ctx) {
String name = null;
String year = null;
for (Text xx : vals) {
String[] parts = xx.toString().split("=");
if (parts[0].equals("name")) {
name = parts[1];
}
if (parts[0].equals("year")) {
year = parts[1];
}
}
try {
if (name != null && year != null) {
ctx.write(new Text(year), new Text(name));
}
} catch (Exception ee) {
ee.printStackTrace(System.err);
throw new Error("I give up");
}
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
Configuration c = context.getConfiguration();
String s = value.toString();
String input[] = s.split(",");
Text outputkey = new Text();
Text outputvalue = new Text();
double result = 0.0;
/* multiplies matrix and vector entry with matching column value */
result = (Double.parseDouble(input[2])) * (vector.get(Long.parseLong(input[1])));
outputkey.set(input[0]);
outputvalue.set(Double.toString(result));
context.write(outputkey, outputvalue);
}
// Identity
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
System.out.println("I'm in Job1 reduce");
for (Text val : values) {
System.out.println("job1:redInp:-" + val.toString());
context.write(new Text(""), val);
}
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
System.out.println("HELLO" + key + value);
String line = value.toString();
String[] components = line.split("\\s+");
context.write(new Text(components[0]), new Text(components[1]));
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String file = value.toString();
String[] lines = file.split("\n");
for (String line : lines) {
if (line.contains("<author>") && line.contains("</author>")) {
String author = line.substring(8, line.indexOf("</a"));
word.set(author);
context.write(word, one);
} else if (line.contains("<author>")) {
String author = line.substring(8);
word.set(author);
context.write(word, one);
}
}
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum = sum + val.get();
}
System.out.println("----------inside-----------");
context.write(key, new Text(Integer.toString(sum)));
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
context.write(word, one);
}
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
Text word = new Text();
StringTokenizer s = new StringTokenizer(value.toString());
while (s.hasMoreTokens()) {
word.set(s.nextToken());
context.write(word, one);
}
}
public void reduce(Text key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (LongWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] currentUserTuples = line.split("::");
int currentAgeValue = Integer.parseInt(currentUserTuples[2].trim());
if (currentUserTuples[1].trim().equalsIgnoreCase("M") && currentAgeValue <= targetAge) {
context.write(new Text("UserId :: " + currentUserTuples[0].trim()), one);
}
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String s = value.toString();
for (String word : s.split("\\W+")) {
if (word.length() > 0) {
context.write(new Text(word), new IntWritable(word.length()));
}
}
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
// read in a document (point in 58-dimensional space)
List<Double> p = GetPoint(value.toString());
int idxClosestCentoid = IndexOfClosestCentroid(p);
context.write(new IntWritable(idxClosestCentoid), value);
}
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
for (Text val : values) {
context.write(key, val);
System.out.println("Pass2 RED KEY: " + key);
System.out.println("Pass2 RED VAL: " + val);
}
}
@Override
public void setup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
int num = Integer.parseInt(conf.get("kvNum"));
while (--num >= 0) {
String s = conf.get(String.valueOf(num));
m_kVectors.add(Kvector.fromString(s));
}
}
@Override
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line);
int movieId = Integer.parseInt(tokenizer.nextToken());
while (tokenizer.hasMoreTokens()) {
String word = tokenizer.nextToken();
context.write(new Text("1"), new IntWritable(1));
}
}
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String input[];
double result = 0.0;
/* adds all the values corresponding to a key */
for (Text value : values) {
result += Double.parseDouble(value.toString());
}
context.write(null, new Text(key + "," + Double.toString(result)));
}
protected void cleanup(Context context) throws IOException, InterruptedException {
Map<Text, IntWritable> sortedMap = sortByValues(countMap);
int counter = 0;
for (Text key : sortedMap.keySet()) {
if (counter++ == 100) {
break;
}
context.write(key, sortedMap.get(key));
}
}
@Override
public void reduce(IdSetterKey key, Iterable<Text> values, Context ctx)
throws IOException, InterruptedException {
if (lineId == -1) {
lineId = calculateFirstIdForReduceOutput(key);
} else {
lineId++;
}
Text outputLine = new Text(key.toString() + "#######" + values.iterator().next().toString());
ctx.write(new LongWritable(lineId), outputLine);
}
/** Reduce task done, write output to a file. */
@Override
public void cleanup(Context context) throws IOException {
// write output to a file
Configuration conf = context.getConfiguration();
Path outDir = new Path(conf.get(FileOutputFormat.OUTDIR));
Path outFile = new Path(outDir, "reduce-out");
FileSystem fileSys = FileSystem.get(conf);
SequenceFile.Writer writer =
SequenceFile.createWriter(
fileSys, conf, outFile, LongWritable.class, LongWritable.class, CompressionType.NONE);
writer.append(new LongWritable(numInside), new LongWritable(numOutside));
writer.close();
}
// Identity mapper (log and write out processed key/value pairs, the value is the
// PersistedEventImpl)
public void map(GFKey key, PersistedEventImpl value, Context context)
throws IOException, InterruptedException {
String keyStr = (String) key.getKey();
Operation op = value.getOperation();
ValueHolder entryValue = null;
System.out.println("map method invoked with " + keyStr + " " + op.toString());
try {
entryValue = (ValueHolder) value.getDeserializedValue();
} catch (ClassNotFoundException e) {
System.out.println("KnownKeysMRv2.map() caught " + e + " : " + TestHelper.getStackTrace(e));
}
context.write(key, new PEIWritable(value));
}
protected void setup(Context context) throws IOException, InterruptedException {
FileSystem fs = FileSystem.get(context.getConfiguration());
Path cFile = new Path(context.getConfiguration().get("CFILE"));
DataInputStream d = new DataInputStream(fs.open(cFile));
BufferedReader reader = new BufferedReader(new InputStreamReader(d));
String line;
while ((line = reader.readLine()) != null) {
StringTokenizer tokenizer = new StringTokenizer(line.toString());
if (tokenizer.hasMoreTokens()) {
List<Double> centroid = new ArrayList<Double>(58);
while (tokenizer.hasMoreTokens()) {
centroid.add(Double.parseDouble(tokenizer.nextToken()));
}
centroids.add(centroid);
}
}
k = centroids.size();
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
splitposition = line.indexOf("\t");
labels = line.substring(0, splitposition);
content = line.substring(splitposition + 1, line.length());
if (content.length() <= 5) {
word.set(labels);
int counter = Integer.parseInt(content);
context.write(word, new IntWritable(counter));
return;
}
labeltokens = labels.split(",");
contenttokens = tokenizeDoc(content);
for (String label : labelspace) {
for (String token : contenttokens) {
// filter token, denotes the needed events;
word.set(label + " " + token);
context.write(word, new IntWritable(-1));
}
}
}