@Override
public void compute(
Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages)
throws IOException {
int currentLength = vertex.getValue().get();
// First superstep is special, because we can simply look at the neighbors
if (getSuperstep() == 0) {
if (isSource(vertex)) {
vertex.setValue(new IntWritable(0));
sendMessageToAllEdges(vertex, vertex.getValue());
} else {
vertex.setValue(new IntWritable(Integer.MAX_VALUE));
}
vertex.voteToHalt();
return;
}
boolean changed = false;
// did we get a smaller length ?
for (IntWritable message : messages) {
int candidateLength = message.get() + 1;
if (candidateLength < currentLength) {
currentLength = candidateLength;
changed = true;
}
}
// propagate new length to the neighbors
if (changed) {
vertex.setValue(new IntWritable(currentLength));
sendMessageToAllEdges(vertex, vertex.getValue());
}
vertex.voteToHalt();
}
@Override
protected void reduce(IntWritable key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
for (IntWritable i : values) {
set.add(i.get());
}
}
public void reduce(
BytesWritable key,
Iterator<IntWritable> values,
OutputCollector<BytesWritable, IntWritable> output,
Reporter reporter)
throws IOException {
int ones = 0;
int twos = 0;
while (values.hasNext()) {
IntWritable count = values.next();
if (count.equals(sortInput)) {
++ones;
} else if (count.equals(sortOutput)) {
++twos;
} else {
throw new IOException(
"Invalid 'value' of " + count.get() + " for (key,value): " + key.toString());
}
}
// Check to ensure there are equal no. of ones and twos
if (ones != twos) {
throw new IOException(
"Illegal ('one', 'two'): ("
+ ones
+ ", "
+ twos
+ ") for (key, value): "
+ key.toString());
}
}
@Override
public void compute(Iterable<IntWritable> messages) {
if (getId().equals(new LongWritable(2))) {
sendMessage(new LongWritable(1), new IntWritable(101));
sendMessage(new LongWritable(1), new IntWritable(102));
sendMessage(new LongWritable(1), new IntWritable(103));
}
if (!getId().equals(new LongWritable(1))) {
voteToHalt();
} else {
// Check the messages
int sum = 0;
int num = 0;
for (IntWritable message : messages) {
sum += message.get();
num++;
}
LOG.info(
"TestCombinerVertex: Received a sum of "
+ sum
+ " (should have 306 with a single message value)");
if (num == 1 && sum == 306) {
voteToHalt();
}
}
if (getSuperstep() > 3) {
throw new IllegalStateException(
"TestCombinerVertex: Vertex 1 failed to receive " + "messages in time");
}
}
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values) sum += value.get();
context.write(key, new IntWritable(sum));
}
@Override
public void compute(
Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages)
throws IOException {
if (0 == getSuperstep()) {
vertex.setValue(new IntWritable(Integer.MAX_VALUE));
}
Integer minDist = isSource(vertex) ? 0 : Integer.MAX_VALUE;
for (IntWritable msg : messages) {
if (msg.get() < minDist) {
minDist = msg.get();
}
}
if (minDist < vertex.getValue().get()) {
vertex.setValue(new IntWritable(minDist));
for (Edge<IntWritable, NullWritable> edge : vertex.getEdges()) {
final IntWritable neighbor = edge.getTargetVertexId();
final Integer distance = minDist + 1;
sendMessage(neighbor, new IntWritable(distance));
}
}
vertex.voteToHalt();
}
@Override
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int ratingsCount = 0;
/*
* For each movieId:
*/
// Define a counter for bad records.
for (IntWritable value : values) {
/*
* Count the movie ratings.
*/
ratingsCount += value.get();
}
/*
* Reducer output is the userID and number of ratings.
*/
context.getCounter(UserIdCounter.NUM_USERS).increment(1);
rValue.set(ratingsCount);
context.write(key, rValue);
}
@Override
protected void map(IntWritable row, VectorWritable similaritiesWritable, Context ctx)
throws IOException, InterruptedException {
Vector similarities = similaritiesWritable.get();
// For performance, the creation of transposedPartial is moved out of the while loop and it is
// reused inside
Vector transposedPartial = new RandomAccessSparseVector(similarities.size(), 1);
TopElementsQueue topKQueue = new TopElementsQueue(maxSimilaritiesPerRow);
Iterator<Vector.Element> nonZeroElements = similarities.iterateNonZero();
while (nonZeroElements.hasNext()) {
Vector.Element nonZeroElement = nonZeroElements.next();
MutableElement top = topKQueue.top();
double candidateValue = nonZeroElement.get();
if (candidateValue > top.get()) {
top.setIndex(nonZeroElement.index());
top.set(candidateValue);
topKQueue.updateTop();
}
transposedPartial.setQuick(row.get(), candidateValue);
ctx.write(new IntWritable(nonZeroElement.index()), new VectorWritable(transposedPartial));
transposedPartial.setQuick(row.get(), 0.0);
}
Vector topKSimilarities =
new RandomAccessSparseVector(similarities.size(), maxSimilaritiesPerRow);
for (Vector.Element topKSimilarity : topKQueue.getTopElements()) {
topKSimilarities.setQuick(topKSimilarity.index(), topKSimilarity.get());
}
ctx.write(row, new VectorWritable(topKSimilarities));
}
@Override
public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) {
size.value = SIZE;
IntWritable writable = new IntWritable();
writable.set(typeConverter.convertTo(Integer.class, value));
return writable;
}
@Override
protected void reduce(IntWritable row, Iterable<VectorWritable> partialDots, Context ctx)
throws IOException, InterruptedException {
Iterator<VectorWritable> partialDotsIterator = partialDots.iterator();
Vector dots = partialDotsIterator.next().get();
while (partialDotsIterator.hasNext()) {
Vector toAdd = partialDotsIterator.next().get();
Iterator<Vector.Element> nonZeroElements = toAdd.iterateNonZero();
while (nonZeroElements.hasNext()) {
Vector.Element nonZeroElement = nonZeroElements.next();
dots.setQuick(
nonZeroElement.index(), dots.getQuick(nonZeroElement.index()) + nonZeroElement.get());
}
}
Vector similarities = dots.like();
double normA = norms.getQuick(row.get());
Iterator<Vector.Element> dotsWith = dots.iterateNonZero();
while (dotsWith.hasNext()) {
Vector.Element b = dotsWith.next();
double similarityValue =
similarity.similarity(b.get(), normA, norms.getQuick(b.index()), numberOfColumns);
if (similarityValue >= treshold) {
similarities.set(b.index(), similarityValue);
}
}
if (excludeSelfSimilarity) {
similarities.setQuick(row.get(), 0);
}
ctx.write(row, new VectorWritable(similarities));
}
@Override
protected void map(IntWritable row, VectorWritable vectorWritable, Context ctx)
throws IOException, InterruptedException {
Vector rowVector = similarity.normalize(vectorWritable.get());
int numNonZeroEntries = 0;
double maxValue = Double.MIN_VALUE;
Iterator<Vector.Element> nonZeroElements = rowVector.iterateNonZero();
while (nonZeroElements.hasNext()) {
Vector.Element element = nonZeroElements.next();
RandomAccessSparseVector partialColumnVector =
new RandomAccessSparseVector(Integer.MAX_VALUE);
partialColumnVector.setQuick(row.get(), element.get());
ctx.write(new IntWritable(element.index()), new VectorWritable(partialColumnVector));
numNonZeroEntries++;
if (maxValue < element.get()) {
maxValue = element.get();
}
}
if (threshold != NO_THRESHOLD) {
nonZeroEntries.setQuick(row.get(), numNonZeroEntries);
maxValues.setQuick(row.get(), maxValue);
}
norms.setQuick(row.get(), similarity.norm(rowVector));
ctx.getCounter(Counters.ROWS).increment(1);
}
public static IntWritable minMsg(Iterable<IntWritable> messages) {
int min = Integer.MAX_VALUE;
for (IntWritable msg : messages) {
min = Math.min(min, msg.get());
}
return new IntWritable(min);
}
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
String uri = args[0];
Configuration conf = new Configuration();
Path path = new Path(uri);
FileSystem fs = FileSystem.get(URI.create(uri), conf);
IntWritable key = new IntWritable();
Text value = new Text();
SequenceFile.Writer writer = null;
try {
writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
for (int i = 0; i < 100; i++) {
key.set(100 - i);
value.set(DATA[i % DATA.length]);
writer.append(key, value);
}
} finally {
IOUtils.closeStream(writer);
}
}
public void run(
RecordReader<IntWritable, WikipediaPage> input,
OutputCollector<IntWritable, Text> output,
Reporter reporter)
throws IOException {
IntWritable key = new IntWritable();
WikipediaPage value = new WikipediaPage();
long pos = -1;
long prevPos = -1;
int prevDocno = 0;
pos = input.getPos();
while (input.next(key, value)) {
if (prevPos != -1 && prevPos != pos) {
LOG.info(
"- beginning of block at " + prevPos + ", docno:" + prevDocno + ", file:" + fileno);
keyOut.set(prevDocno);
valOut.set(prevPos + "\t" + fileno);
output.collect(keyOut, valOut);
reporter.incrCounter(Blocks.Total, 1);
}
prevPos = pos;
pos = input.getPos();
prevDocno = key.get();
}
}
public void map(
LongWritable key,
Text value,
OutputCollector<IntWritable, HITSNode> output,
Reporter reporter)
throws IOException {
ArrayListOfIntsWritable links = new ArrayListOfIntsWritable();
String line = ((Text) value).toString();
StringTokenizer itr = new StringTokenizer(line);
if (itr.hasMoreTokens()) {
int curr = Integer.parseInt(itr.nextToken());
if (stopList.contains(curr)) {
return;
}
valOut.setAdjacencyList(links);
valOut.setHARank((float) 1.0);
valOut.setType(HITSNode.TYPE_AUTH_COMPLETE);
}
while (itr.hasMoreTokens()) {
keyOut.set(Integer.parseInt(itr.nextToken()));
valOut.setNodeId(keyOut.get());
// System.out.println(keyOut.toString() + ", " +
// valOut.toString());
if (!(stopList.contains(keyOut.get()))) {
output.collect(keyOut, valOut);
}
}
// emit mentioned mentioner -> mentioned (mentioners) in links
// emit mentioner mentioned -> mentioner (mentions) outlinks
// emit mentioned a
// emit mentioner 1
}
@Before
public void setUp() throws Exception {
// create local Pig server
pigServer = UnitTestUtil.makePigServer();
// create temp SequenceFile
File tempFile = File.createTempFile("test", ".txt");
tempFilename = tempFile.getAbsolutePath();
Path path = new Path("file:///" + tempFilename);
Configuration conf = new Configuration();
FileSystem fs = path.getFileSystem(conf);
IntWritable key = new IntWritable();
Text value = new Text();
SequenceFile.Writer writer = null;
try {
writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
for (int i = 0; i < DATA.length; ++i) {
key.set(i);
value.set(DATA[i]);
writer.append(key, value);
}
} finally {
IOUtils.closeStream(writer);
}
}
public static double getAsConstDouble(@Nonnull final ObjectInspector numberOI)
throws UDFArgumentException {
final String typeName = numberOI.getTypeName();
if (DOUBLE_TYPE_NAME.equals(typeName)) {
DoubleWritable v = getConstValue(numberOI);
return v.get();
} else if (FLOAT_TYPE_NAME.equals(typeName)) {
FloatWritable v = getConstValue(numberOI);
return v.get();
} else if (INT_TYPE_NAME.equals(typeName)) {
IntWritable v = getConstValue(numberOI);
return v.get();
} else if (BIGINT_TYPE_NAME.equals(typeName)) {
LongWritable v = getConstValue(numberOI);
return v.get();
} else if (SMALLINT_TYPE_NAME.equals(typeName)) {
ShortWritable v = getConstValue(numberOI);
return v.get();
} else if (TINYINT_TYPE_NAME.equals(typeName)) {
ByteWritable v = getConstValue(numberOI);
return v.get();
}
throw new UDFArgumentException(
"Unexpected argument type to cast as double: "
+ TypeInfoUtils.getTypeInfoFromObjectInspector(numberOI));
}
/**
* @param state The final LanczosState to be serialized
* @param outputPath The path (relative to the current Configuration's FileSystem) to save the
* output to.
*/
public void serializeOutput(LanczosState state, Path outputPath) throws IOException {
int numEigenVectors = state.getIterationNumber();
log.info("Persisting {} eigenVectors and eigenValues to: {}", numEigenVectors, outputPath);
Configuration conf = getConf() != null ? getConf() : new Configuration();
FileSystem fs = FileSystem.get(outputPath.toUri(), conf);
SequenceFile.Writer seqWriter =
new SequenceFile.Writer(fs, conf, outputPath, IntWritable.class, VectorWritable.class);
try {
IntWritable iw = new IntWritable();
for (int i = 0; i < numEigenVectors; i++) {
// Persist eigenvectors sorted by eigenvalues in descending order\
NamedVector v =
new NamedVector(
state.getRightSingularVector(numEigenVectors - 1 - i),
"eigenVector"
+ i
+ ", eigenvalue = "
+ state.getSingularValue(numEigenVectors - 1 - i));
Writable vw = new VectorWritable(v);
iw.set(i);
seqWriter.append(iw, vw);
}
} finally {
Closeables.close(seqWriter, false);
}
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
boolean filtered = true;
String keypair = key.toString();
String[] keys = keypair.split(" ");
if (keys.length == 1) {
filtered = false;
} else {
if (keys[0].equals("*") || keys[1].equals("*")) {
filtered = false;
}
}
if (!filtered) {
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
return;
}
for (IntWritable val : values) {
if (val.get() == -1) {
filtered = false;
continue;
}
sum += val.get();
}
// filter non-needed events
if (filtered) return;
context.write(key, new IntWritable(sum));
}
@Override
public int compareTo(KeyPair otherPair) {
// TODO Auto-generated method stub
int c = ArivalDelay.compareTo(otherPair.ArivalDelay);
if (c != 0) return c;
else return Flightnumber.compareTo(otherPair.Flightnumber);
}
@Override
public boolean equals(Object o) {
if (o instanceof PairIntWritable) {
PairIntWritable tp = (PairIntWritable) o;
return first.equals(tp.first) && second.equals(tp.second);
}
return false;
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int maxValues = Integer.MIN_VALUE;
for (IntWritable value : values) {
maxValues = Math.max(maxValues, value.get());
}
context.write(key, new IntWritable(maxValues));
}
public static int getConstInt(@Nonnull final ObjectInspector oi) throws UDFArgumentException {
if (!isIntOI(oi)) {
throw new UDFArgumentException(
"argument must be a Int value: " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi));
}
IntWritable v = getConstValue(oi);
return v.get();
}
@Override
public int compareTo(PairIntWritable tp) {
int cmp = first.compareTo(tp.first);
if (cmp != 0) {
return cmp;
}
return second.compareTo(tp.second);
}
public void reduce(BytesWritable key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
public void reduce(Text key, Iterable<IntWritable> results, Context context)
throws IOException, InterruptedException {
int total = 0;
for (IntWritable cost : results) {
total += cost.get();
}
context.write(key, new IntWritable(total));
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int i = 0;
for (IntWritable val : values) {
i += val.get();
}
context.write(key, new IntWritable(i));
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
countMap.put(new Text(key), new IntWritable(sum));
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values) {
sum += value.get();
}
reducedValue.set(sum);
context.write(key, new IntWritable(sum));
}
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int totalFreq = 0;
for (IntWritable value : values) {
totalFreq = totalFreq + value.get();
}
context.write(key, new IntWritable(totalFreq));
}