public IntWritable evaluate(Text s) {
if (s == null) {
return null;
}
if (s.getLength() > 0) {
result.set(s.getBytes()[0]);
} else {
result.set(0);
}
return result;
}
public void map(
LongWritable key,
Text value,
OutputCollector<IntWritable, IntWritable> output,
Reporter reporter)
throws IOException {
String line = value.toString();
IntWritable clave = new IntWritable();
IntWritable valor = new IntWritable();
clave.set(Integer.parseInt(line));
valor.set(Integer.parseInt(line) + 1);
output.collect(clave, valor);
}
public void map(
Text key, Text value, OutputCollector<IntWritable, IntWritable> output, Reporter reporter)
throws IOException {
citationCount.set(Integer.parseInt(value.toString()));
output.collect(citationCount, uno);
}
public void map(
LongWritable key,
Text value,
OutputCollector<IntWritable, DoubleWritable> output,
Reporter reporter)
throws IOException {
/*
* It implements the mapper. It outputs the numbers of weight and updated weights.
*
* Note that the format of intermediate output is <IntWritable, DoubleWritable>,
* because the key is the number of weight (an integer), and the value is the weight's value (double)
*/
inputData = value.toString();
// go through the process
initialize();
getposphase();
getnegphase();
update();
// output the intermediate data
// The <key, value> pairs are <weightID, weightUpdate>
double[][] vishidinc_array = vishidinc.getArray();
for (int i = 0; i < numdims; i++) {
for (int j = 0; j < numhid; j++) {
weightPos.set(i * numhid + j);
weightValue.set(vishidinc_array[i][j]);
output.collect(weightPos, weightValue);
}
}
}
public void run(
RecordReader<IntWritable, WikipediaPage> input,
OutputCollector<IntWritable, Text> output,
Reporter reporter)
throws IOException {
IntWritable key = new IntWritable();
WikipediaPage value = new WikipediaPage();
long pos = -1;
long prevPos = -1;
int prevDocno = 0;
pos = input.getPos();
while (input.next(key, value)) {
if (prevPos != -1 && prevPos != pos) {
LOG.info(
"- beginning of block at " + prevPos + ", docno:" + prevDocno + ", file:" + fileno);
keyOut.set(prevDocno);
valOut.set(prevPos + "\t" + fileno);
output.collect(keyOut, valOut);
reporter.incrCounter(Blocks.Total, 1);
}
prevPos = pos;
pos = input.getPos();
prevDocno = key.get();
}
}
@Override
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int ratingsCount = 0;
/*
* For each movieId:
*/
// Define a counter for bad records.
for (IntWritable value : values) {
/*
* Count the movie ratings.
*/
ratingsCount += value.get();
}
/*
* Reducer output is the userID and number of ratings.
*/
context.getCounter(UserIdCounter.NUM_USERS).increment(1);
rValue.set(ratingsCount);
context.write(key, rValue);
}
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
System.out.println("isitlinewise2: " + value.toString());
StringTokenizer itr = new StringTokenizer(value.toString());
System.out.println(value);
// 1st token is the key, you don't want to process it
itr.nextToken();
int worth = 0;
List<String> valList = new ArrayList<String>();
System.out.println("map1----" + key);
while (itr.hasMoreTokens()) {
String val = itr.nextToken();
System.out.println("map1 vals1: " + val.toString());
try {
worth = Integer.parseInt(val.toString());
System.out.println("map1 worth: " + worth);
} catch (NumberFormatException e) {
valList.add(val.toString());
}
}
for (String val : valList) {
System.out.println("map1 vals2: " + val.toString() + " " + worth);
word.set(val);
wordWorth.set(worth);
context.write(word, wordWorth);
}
}
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
data.set(Integer.parseInt(line));
context.write(data, new IntWritable(1));
}
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
String uri = args[0];
Configuration conf = new Configuration();
Path path = new Path(uri);
FileSystem fs = FileSystem.get(URI.create(uri), conf);
IntWritable key = new IntWritable();
Text value = new Text();
SequenceFile.Writer writer = null;
try {
writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
for (int i = 0; i < 100; i++) {
key.set(100 - i);
value.set(DATA[i % DATA.length]);
writer.append(key, value);
}
} finally {
IOUtils.closeStream(writer);
}
}
public IntWritable readInt(IntWritable iw) throws IOException {
if (iw == null) {
iw = new IntWritable();
}
iw.set(in.readInt());
return iw;
}
public void map(
LongWritable key,
Text value,
OutputCollector<IntWritable, HITSNode> output,
Reporter reporter)
throws IOException {
ArrayListOfIntsWritable links = new ArrayListOfIntsWritable();
String line = ((Text) value).toString();
StringTokenizer itr = new StringTokenizer(line);
if (itr.hasMoreTokens()) {
int curr = Integer.parseInt(itr.nextToken());
if (stopList.contains(curr)) {
return;
}
valOut.setAdjacencyList(links);
valOut.setHARank((float) 1.0);
valOut.setType(HITSNode.TYPE_AUTH_COMPLETE);
}
while (itr.hasMoreTokens()) {
keyOut.set(Integer.parseInt(itr.nextToken()));
valOut.setNodeId(keyOut.get());
// System.out.println(keyOut.toString() + ", " +
// valOut.toString());
if (!(stopList.contains(keyOut.get()))) {
output.collect(keyOut, valOut);
}
}
// emit mentioned mentioner -> mentioned (mentioners) in links
// emit mentioner mentioned -> mentioner (mentions) outlinks
// emit mentioned a
// emit mentioner 1
}
public void map(
LongWritable key,
Text t,
OutputCollector<IntWritable, PageRankNode> output,
Reporter reporter)
throws IOException {
String[] arr = t.toString().trim().split("\\s+");
nid.set(Integer.parseInt(arr[0]));
if (arr.length == 1) {
node.setNodeId(Integer.parseInt(arr[0]));
node.setAdjacencyList(new ArrayListOfIntsWritable());
} else {
node.setNodeId(Integer.parseInt(arr[0]));
int[] neighbors = new int[arr.length - 1];
for (int i = 1; i < arr.length; i++) {
neighbors[i - 1] = Integer.parseInt(arr[i]);
}
node.setAdjacencyList(new ArrayListOfIntsWritable(neighbors));
}
reporter.incrCounter("graph", "numNodes", 1);
reporter.incrCounter("graph", "numEdges", arr.length - 1);
if (arr.length > 1) {
reporter.incrCounter("graph", "numActiveNodes", 1);
}
output.collect(nid, node);
}
@Before
public void setUp() throws Exception {
// create local Pig server
pigServer = UnitTestUtil.makePigServer();
// create temp SequenceFile
File tempFile = File.createTempFile("test", ".txt");
tempFilename = tempFile.getAbsolutePath();
Path path = new Path("file:///" + tempFilename);
Configuration conf = new Configuration();
FileSystem fs = path.getFileSystem(conf);
IntWritable key = new IntWritable();
Text value = new Text();
SequenceFile.Writer writer = null;
try {
writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
for (int i = 0; i < DATA.length; ++i) {
key.set(i);
value.set(DATA[i]);
writer.append(key, value);
}
} finally {
IOUtils.closeStream(writer);
}
}
public IntWritable evaluate(Text s) {
if (s == null) {
return null;
}
result.set(s.toString().length());
return result;
}
/**
* @param state The final LanczosState to be serialized
* @param outputPath The path (relative to the current Configuration's FileSystem) to save the
* output to.
*/
public void serializeOutput(LanczosState state, Path outputPath) throws IOException {
int numEigenVectors = state.getIterationNumber();
log.info("Persisting {} eigenVectors and eigenValues to: {}", numEigenVectors, outputPath);
Configuration conf = getConf() != null ? getConf() : new Configuration();
FileSystem fs = FileSystem.get(outputPath.toUri(), conf);
SequenceFile.Writer seqWriter =
new SequenceFile.Writer(fs, conf, outputPath, IntWritable.class, VectorWritable.class);
try {
IntWritable iw = new IntWritable();
for (int i = 0; i < numEigenVectors; i++) {
// Persist eigenvectors sorted by eigenvalues in descending order\
NamedVector v =
new NamedVector(
state.getRightSingularVector(numEigenVectors - 1 - i),
"eigenVector"
+ i
+ ", eigenvalue = "
+ state.getSingularValue(numEigenVectors - 1 - i));
Writable vw = new VectorWritable(v);
iw.set(i);
seqWriter.append(iw, vw);
}
} finally {
Closeables.close(seqWriter, false);
}
}
public void cleanup(Context context) throws IOException, InterruptedException { // close
for (Entry<String, Integer> entry : wordMap.entrySet()) {
word.set(entry.getKey());
counter.set(entry.getValue());
context.write(word, counter);
}
}
@Override
public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) {
size.value = SIZE;
IntWritable writable = new IntWritable();
writable.set(typeConverter.convertTo(Integer.class, value));
return writable;
}
/**
* Convert from long to an integer. This is called for CAST(... AS INT)
*
* @param i The long value to convert
* @return IntWritable
*/
public IntWritable evaluate(LongWritable i) {
if (i == null) {
return null;
} else {
intWritable.set((int) i.get());
return intWritable;
}
}
/**
* Convert from Timestamp to an integer. This is called for CAST(... AS INT)
*
* @param i The Timestamp value to convert
* @return IntWritable
*/
public IntWritable evaluate(TimestampWritable i) {
if (i == null) {
return null;
} else {
intWritable.set(i.getSeconds());
return intWritable;
}
}
public IntWritable evaluate(HiveIntervalYearMonthWritable i) {
if (i == null) {
return null;
}
result.set(i.getHiveIntervalYearMonth().getMonths());
return result;
}
/**
* Convert from boolean to an integer. This is called for CAST(... AS INT)
*
* @param i The boolean value to convert
* @return IntWritable
*/
public IntWritable evaluate(BooleanWritable i) {
if (i == null) {
return null;
} else {
intWritable.set(i.get() ? 1 : 0);
return intWritable;
}
}
/**
* Convert from short to an integer. This is called for CAST(... AS INT)
*
* @param i The short value to convert
* @return IntWritable
*/
public IntWritable evaluate(ShortWritable i) {
if (i == null) {
return null;
} else {
intWritable.set(i.get());
return intWritable;
}
}
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values) sum += value.get();
writableSum.set(sum);
context.write(key, writableSum);
}
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line[] = value.toString().split("[\\s]");
word.set(line[0].trim());
freq.set(-1 * Integer.parseInt(line[1].trim()));
context.write(freq, word);
}
public IntWritable evaluate(DateWritable d) {
if (d == null) {
return null;
}
calendar.setTime(d.get());
result.set(1 + calendar.get(Calendar.MONTH));
return result;
}
public IntWritable evaluate(TimestampWritable t) {
if (t == null) {
return null;
}
calendar.setTime(t.getTimestamp());
result.set(1 + calendar.get(Calendar.MONTH));
return result;
}
@Override
protected void reduce(IntWritable key, Iterable<Text> value, Context context)
throws IOException, InterruptedException {
for (Text val : value) {
Freq.set(-1 * (key.get()));
context.write(val, Freq);
}
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values) {
sum += value.get();
}
reducedValue.set(sum);
context.write(key, new IntWritable(sum));
}
public void reduce(Key key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
public void close() throws IOException {
for (MapIV.Entry<ArrayListOfIntsWritable> e : adjLists.entrySet()) {
keyOut.set(e.getKey());
valOut.setNodeId(e.getKey());
valOut.setHARank((float) 0.0);
valOut.setType(HITSNode.TYPE_AUTH_COMPLETE);
valOut.setAdjacencyList(e.getValue());
mOutput.collect(keyOut, valOut);
}
}
