public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String cur_file =
((FileSplit) context.getInputSplit()).getPath().getParent().getParent().getName();
String train_file = context.getConfiguration().get("train_file");
if (cur_file.equals(train_file)) {
StringTokenizer st = new StringTokenizer(value.toString());
String word = st.nextToken();
String f_id = st.nextToken();
myKey.set(word);
myVal.set(f_id);
context.write(myKey, myVal);
} else {
StringTokenizer st = new StringTokenizer(value.toString());
String word = st.nextToken();
String f_id = st.nextToken();
StringBuilder builder = new StringBuilder(dlt);
while (st.hasMoreTokens()) {
String filename = st.nextToken();
String tf_idf = st.nextToken();
builder.append(filename);
builder.append(dlt);
builder.append(tf_idf);
builder.append("\t");
}
myKey.set(word);
myVal.set(builder.toString());
context.write(myKey, myVal);
}
}
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String keyS = key.toString();
if (keyS.startsWith("O") || keyS.startsWith("P") || keyS.startsWith("S")) {
String sum = new String();
for (Text val : values) {
sum += (" " + val.toString());
}
// String subKey = keyS.substring(0,keyS.length()-1);
// Text t = new Text();
// t.set(subKey);
result.set(sum);
context.write(key, result);
}
if (keyS.startsWith("L")) {
// String [] keyIdS = keyS.substring(1).split("[+]");
result.set(" ");
context.write(key, result);
// String KeyIdS1 = keyIdS[1];
// result.set(KeyIdS1);
// context.write(key, result);
// String KeyIdS2 = keyIdS[2];
// result.set(KeyIdS2);
// context.write(key, result);
}
}
@Override
public void reduce(IntWritable key, Iterable<WriteableData> values, Context context)
throws IOException, InterruptedException {
DaalContext daalContext = new DaalContext();
/* Create an algorithm to compute a sparse variance-covariance matrix on the master node */
DistributedStep2Master covarianceSparseMaster =
new DistributedStep2Master(daalContext, Double.class, Method.fastCSR);
for (WriteableData value : values) {
PartialResult pr = (PartialResult) value.getObject(daalContext);
covarianceSparseMaster.input.add(DistributedStep2MasterInputId.partialResults, pr);
}
/* Compute a sparse variance-covariance matrix on the master node */
covarianceSparseMaster.compute();
/* Finalize computations and retrieve the results */
Result result = covarianceSparseMaster.finalizeCompute();
HomogenNumericTable covariance = (HomogenNumericTable) result.get(ResultId.covariance);
HomogenNumericTable mean = (HomogenNumericTable) result.get(ResultId.mean);
context.write(new IntWritable(0), new WriteableData(covariance));
context.write(new IntWritable(1), new WriteableData(mean));
daalContext.dispose();
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
boolean filtered = true;
String keypair = key.toString();
String[] keys = keypair.split(" ");
if (keys.length == 1) {
filtered = false;
} else {
if (keys[0].equals("*") || keys[1].equals("*")) {
filtered = false;
}
}
if (!filtered) {
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
return;
}
for (IntWritable val : values) {
if (val.get() == -1) {
filtered = false;
continue;
}
sum += val.get();
}
// filter non-needed events
if (filtered) return;
context.write(key, new IntWritable(sum));
}
/*
* Setup gets called exactly once for each mapper, before map() gets called the first time.
* It's a good place to do configuration or setup that can be shared across many calls to map
*/
@Override
public void setup(Context context) {
targetGram = context.getConfiguration().get("targetWord").toLowerCase();
try {
funcNum = Integer.parseInt(context.getConfiguration().get("funcNum"));
} catch (NumberFormatException e) {
/* Do nothing. */
}
}
public void reduce(GFKey key, Iterable<PEIWritable> values, Context context)
throws IOException, InterruptedException {
// For a particular key ... process all records and output what we would have expected in this
// concKnownKeys test
// Note that we either
// 1. do a single create
// 2. create + update
// 3. create + destroy
// look at all ops ... and output either
// 1. create
// 2. create (with value from update)
// 3. do nothing (overall result is destroy, so do not create the entry in the gemfire
// validation region
String keyStr = (String) key.getKey();
ValueHolder updateValue = null;
ValueHolder createValue = null;
boolean destroyed = false;
System.out.println("KnownKeysMRv2.reduce() invoked with " + keyStr);
for (PEIWritable value : values) {
PersistedEventImpl event = value.getEvent();
Operation op = event.getOperation();
ValueHolder vh = null;
if (op.isDestroy()) {
destroyed = true;
} else {
try {
vh = (ValueHolder) event.getDeserializedValue();
} catch (ClassNotFoundException e) {
System.out.println(
"KnownKeysMRv2.map() caught " + e + " : " + TestHelper.getStackTrace(e));
}
if (op.isUpdate()) {
updateValue = vh;
} else {
createValue = vh;
}
}
System.out.println(
"KnownKeysMRv2.reduce() record: "
+ op.toString()
+ ": key = "
+ keyStr
+ " and op "
+ op.toString());
}
if (!destroyed) {
if (updateValue != null) {
context.write(key.getKey(), updateValue);
} else {
context.write(key.getKey(), createValue);
}
}
}
private static void WriteNewCentroids(Context context) throws IOException {
FileSystem fs = FileSystem.get(context.getConfiguration());
Path nextCFile = new Path(context.getConfiguration().get("NEXTCFILE"));
DataOutputStream d = new DataOutputStream(fs.create(nextCFile, false));
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(d));
for (String centroid : newCentroids) {
writer.write(centroid + "\n");
}
writer.close();
}
public void reduce(LongWritable key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
LongWritable curNodeId = key;
double previousPRValue = 1;
double nextPRValue = 0;
double localResidual = 0;
String edgeListOfCurNode = "";
long localResidualTransformed = 0;
for (Text value : values) {
String[] inputInfo = value.toString().split("\\s+");
// incoming pagerank value
if (inputInfo.length == 1) {
nextPRValue += Double.parseDouble(inputInfo[0]);
}
// current node info
else if (inputInfo.length == 3) {
edgeListOfCurNode = inputInfo[2];
previousPRValue = Double.parseDouble(inputInfo[1]);
} else if (inputInfo.length == 2) {
previousPRValue = Double.parseDouble(inputInfo[1]);
} else {
System.out.println("ERROR: received unexpected TEXT in length");
}
}
if (previousPRValue == 1) System.out.println("No node info has been received by a reducer");
// calculate the pagerank value according to the given formula
nextPRValue = pagerankFormula(nextPRValue);
// should also iterate sink nodes list, add the evenly splitted value
// reducer should store the updated node info(NPR) to output directory
context.write(null, new Text(curNodeId + " " + nextPRValue + " " + edgeListOfCurNode));
// then compare PPR with NPR
try {
localResidual = Math.abs(previousPRValue - nextPRValue) / nextPRValue;
localResidualTransformed = (long) (localResidual * 10000);
// System.out.println("Make sure you got the right transformed residual :
// "+localResidualTransformed);
} catch (ArithmeticException e) {
System.out.println("PPR is zero. Check where you get the value!");
}
// assume there is a global counter called residualCounter;
context.getCounter(myCounter.ResidualCounter.RESIDUAL_SUM).increment(localResidualTransformed);
}
@Override
protected void reduce(Text key, Iterable<Text> vals, Context ctx) {
String name = null;
String year = null;
for (Text xx : vals) {
String[] parts = xx.toString().split("=");
if (parts[0].equals("name")) {
name = parts[1];
}
if (parts[0].equals("year")) {
year = parts[1];
}
}
try {
if (name != null && year != null) {
ctx.write(new Text(year), new Text(name));
}
} catch (Exception ee) {
ee.printStackTrace(System.err);
throw new Error("I give up");
}
}
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line, " ");
int dim = tokenizer.countTokens();
double[] coords = new double[dim];
for (int i = 0; i < dim; i++) {
coords[i] = Double.valueOf(tokenizer.nextToken());
}
Point point = new Point(coords);
Centroid nearest = null;
double nearestDistance = Double.MAX_VALUE;
for (Centroid c : centers) {
double dist = point.euclideanDistance(c);
if (nearest == null) {
nearest = c;
nearestDistance = dist;
} else {
if (dist < nearestDistance) {
nearest = c;
nearestDistance = dist;
}
}
}
context.write(nearest, point);
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
// TODO
// TextArrayWritable tmp;
// while(values.hasNext()){
for (TextArrayWritable value : values) {
for (Writable w : value.get()) {
String str = ((Text) w).toString();
String arrStr[] = str.split("\\|");
tree.add(new Pair(Integer.parseInt(arrStr[0]), arrStr[1]));
}
}
// reverse tree set
TreeSet<Pair> treereverse = new TreeSet<Pair>();
treereverse = (TreeSet) tree.descendingSet();
Iterator itr = treereverse.iterator();
int count = 1;
while (itr.hasNext() && count != N + 1) {
Pair p = (Pair) itr.next();
// send top N entry
context.write(
new Text(p.second.toString()), new IntWritable(Integer.parseInt(p.first.toString())));
count++;
}
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
Configuration c = context.getConfiguration();
String s = value.toString();
String input[] = s.split(",");
Text outputkey = new Text();
Text outputvalue = new Text();
double result = 0.0;
/* multiplies matrix and vector entry with matching column value */
result = (Double.parseDouble(input[2])) * (vector.get(Long.parseLong(input[1])));
outputkey.set(input[0]);
outputvalue.set(Double.toString(result));
context.write(outputkey, outputvalue);
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
super.cleanup(context);
Configuration conf = context.getConfiguration();
Path outPath = new Path(conf.get(CENTERS_CONF_KEY));
FileSystem fs = FileSystem.get(conf);
// fs.delete(outPath, true);
SequenceFile.Writer writer =
SequenceFile.createWriter(
fs, context.getConfiguration(), outPath, Centroid.class, IntWritable.class);
final IntWritable mockValue = new IntWritable(0);
for (Centroid center : centers) {
writer.append(center, mockValue);
}
writer.close();
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
System.out.println("HELLO" + key + value);
String line = value.toString();
String[] components = line.split("\\s+");
context.write(new Text(components[0]), new Text(components[1]));
}
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String file = value.toString();
String[] lines = file.split("\n");
for (String line : lines) {
if (line.contains("<author>") && line.contains("</author>")) {
String author = line.substring(8, line.indexOf("</a"));
word.set(author);
context.write(word, one);
} else if (line.contains("<author>")) {
String author = line.substring(8);
word.set(author);
context.write(word, one);
}
}
}
/**
* Emit all the values for the final solved values table. We want to print this out with the
* expanded string representation of the game board (to make the output file easily human
* readable and thus easier to debug).
*/
@Override
public void reduce(IntWritable key, Iterable<ByteWritable> values, Context context)
throws IOException, InterruptedException {
int keyInt = key.get();
if (keyInt == 0) return;
String hash = "'" + Proj2Util.gameUnhasher(keyInt, boardWidth, boardHeight) + "'";
for (ByteWritable val : values) context.write(new Text(hash), val);
}
// Identity
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
System.out.println("I'm in Job1 reduce");
for (Text val : values) {
System.out.println("job1:redInp:-" + val.toString());
context.write(new Text(""), val);
}
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
/**
* Emits provided key-value pair as long as the value field of the MovesWritable is non-zero.
*/
@Override
public void map(IntWritable key, MovesWritable val, Context context)
throws IOException, InterruptedException {
byte value = val.getValue();
if (value == 0) {
return;
}
context.write(key, new ByteWritable(value));
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
Text word = new Text();
StringTokenizer s = new StringTokenizer(value.toString());
while (s.hasMoreTokens()) {
word.set(s.nextToken());
context.write(word, one);
}
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] currentUserTuples = line.split("::");
int currentAgeValue = Integer.parseInt(currentUserTuples[2].trim());
if (currentUserTuples[1].trim().equalsIgnoreCase("M") && currentAgeValue <= targetAge) {
context.write(new Text("UserId :: " + currentUserTuples[0].trim()), one);
}
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum = sum + val.get();
}
System.out.println("----------inside-----------");
context.write(key, new Text(Integer.toString(sum)));
}
public void reduce(Text key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (LongWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
context.write(word, one);
}
}
/**
* Maps each number into an initial key/value pair This method will ignore any non-numbers (e.g.
* letters/grammar)
*
* @param lwKey initial key
* @param tValue initial value
* @param cContext initial context
* @throws IOException
*/
@Override
public void map(LongWritable lwKey, Text tValue, Context cContext) throws IOException {
String line = tValue.toString();
cContext.getConfiguration().getStrings(line);
String[] lineArr = line.split("\n");
for (String string : lineArr) {
if (string.matches("-?\\d+(\\.\\d+)?")) {
tWord.set(string);
try {
cContext.write(tWord, iwOne);
} catch (InterruptedException ex) {
Logger.getLogger(FrequencyTest.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
for (Text val : values) {
context.write(key, val);
System.out.println("Pass2 RED KEY: " + key);
System.out.println("Pass2 RED VAL: " + val);
}
}
@Override
public void setup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
int num = Integer.parseInt(conf.get("kvNum"));
while (--num >= 0) {
String s = conf.get(String.valueOf(num));
m_kVectors.add(Kvector.fromString(s));
}
}
@Override
/** (K1, V1) = (line#, line) (K2, V2) = (visitor full name, 1) */
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
parser.parse(value);
if (parser.isValidRecord()) {
context.write(new Text(parser.getVisitorFullname()), one);
}
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
// read in a document (point in 58-dimensional space)
List<Double> p = GetPoint(value.toString());
int idxClosestCentoid = IndexOfClosestCentroid(p);
context.write(new IntWritable(idxClosestCentoid), value);
}
@Override
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line);
int movieId = Integer.parseInt(tokenizer.nextToken());
while (tokenizer.hasMoreTokens()) {
String word = tokenizer.nextToken();
context.write(new Text("1"), new IntWritable(1));
}
}