@Override public void compute( Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages) throws IOException { int currentLength = vertex.getValue().get(); // First superstep is special, because we can simply look at the neighbors if (getSuperstep() == 0) { if (isSource(vertex)) { vertex.setValue(new IntWritable(0)); sendMessageToAllEdges(vertex, vertex.getValue()); } else { vertex.setValue(new IntWritable(Integer.MAX_VALUE)); } vertex.voteToHalt(); return; } boolean changed = false; // did we get a smaller length ? for (IntWritable message : messages) { int candidateLength = message.get() + 1; if (candidateLength < currentLength) { currentLength = candidateLength; changed = true; } } // propagate new length to the neighbors if (changed) { vertex.setValue(new IntWritable(currentLength)); sendMessageToAllEdges(vertex, vertex.getValue()); } vertex.voteToHalt(); }
@Override protected void reduce(IntWritable key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { for (IntWritable i : values) { set.add(i.get()); } }
public void reduce( BytesWritable key, Iterator<IntWritable> values, OutputCollector<BytesWritable, IntWritable> output, Reporter reporter) throws IOException { int ones = 0; int twos = 0; while (values.hasNext()) { IntWritable count = values.next(); if (count.equals(sortInput)) { ++ones; } else if (count.equals(sortOutput)) { ++twos; } else { throw new IOException( "Invalid 'value' of " + count.get() + " for (key,value): " + key.toString()); } } // Check to ensure there are equal no. of ones and twos if (ones != twos) { throw new IOException( "Illegal ('one', 'two'): (" + ones + ", " + twos + ") for (key, value): " + key.toString()); } }
@Override public void compute(Iterable<IntWritable> messages) { if (getId().equals(new LongWritable(2))) { sendMessage(new LongWritable(1), new IntWritable(101)); sendMessage(new LongWritable(1), new IntWritable(102)); sendMessage(new LongWritable(1), new IntWritable(103)); } if (!getId().equals(new LongWritable(1))) { voteToHalt(); } else { // Check the messages int sum = 0; int num = 0; for (IntWritable message : messages) { sum += message.get(); num++; } LOG.info( "TestCombinerVertex: Received a sum of " + sum + " (should have 306 with a single message value)"); if (num == 1 && sum == 306) { voteToHalt(); } } if (getSuperstep() > 3) { throw new IllegalStateException( "TestCombinerVertex: Vertex 1 failed to receive " + "messages in time"); } }
@Override protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable value : values) sum += value.get(); context.write(key, new IntWritable(sum)); }
@Override public void compute( Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages) throws IOException { if (0 == getSuperstep()) { vertex.setValue(new IntWritable(Integer.MAX_VALUE)); } Integer minDist = isSource(vertex) ? 0 : Integer.MAX_VALUE; for (IntWritable msg : messages) { if (msg.get() < minDist) { minDist = msg.get(); } } if (minDist < vertex.getValue().get()) { vertex.setValue(new IntWritable(minDist)); for (Edge<IntWritable, NullWritable> edge : vertex.getEdges()) { final IntWritable neighbor = edge.getTargetVertexId(); final Integer distance = minDist + 1; sendMessage(neighbor, new IntWritable(distance)); } } vertex.voteToHalt(); }
@Override public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int ratingsCount = 0; /* * For each movieId: */ // Define a counter for bad records. for (IntWritable value : values) { /* * Count the movie ratings. */ ratingsCount += value.get(); } /* * Reducer output is the userID and number of ratings. */ context.getCounter(UserIdCounter.NUM_USERS).increment(1); rValue.set(ratingsCount); context.write(key, rValue); }
@Override protected void map(IntWritable row, VectorWritable similaritiesWritable, Context ctx) throws IOException, InterruptedException { Vector similarities = similaritiesWritable.get(); // For performance, the creation of transposedPartial is moved out of the while loop and it is // reused inside Vector transposedPartial = new RandomAccessSparseVector(similarities.size(), 1); TopElementsQueue topKQueue = new TopElementsQueue(maxSimilaritiesPerRow); Iterator<Vector.Element> nonZeroElements = similarities.iterateNonZero(); while (nonZeroElements.hasNext()) { Vector.Element nonZeroElement = nonZeroElements.next(); MutableElement top = topKQueue.top(); double candidateValue = nonZeroElement.get(); if (candidateValue > top.get()) { top.setIndex(nonZeroElement.index()); top.set(candidateValue); topKQueue.updateTop(); } transposedPartial.setQuick(row.get(), candidateValue); ctx.write(new IntWritable(nonZeroElement.index()), new VectorWritable(transposedPartial)); transposedPartial.setQuick(row.get(), 0.0); } Vector topKSimilarities = new RandomAccessSparseVector(similarities.size(), maxSimilaritiesPerRow); for (Vector.Element topKSimilarity : topKQueue.getTopElements()) { topKSimilarities.setQuick(topKSimilarity.index(), topKSimilarity.get()); } ctx.write(row, new VectorWritable(topKSimilarities)); }
@Override public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) { size.value = SIZE; IntWritable writable = new IntWritable(); writable.set(typeConverter.convertTo(Integer.class, value)); return writable; }
@Override protected void reduce(IntWritable row, Iterable<VectorWritable> partialDots, Context ctx) throws IOException, InterruptedException { Iterator<VectorWritable> partialDotsIterator = partialDots.iterator(); Vector dots = partialDotsIterator.next().get(); while (partialDotsIterator.hasNext()) { Vector toAdd = partialDotsIterator.next().get(); Iterator<Vector.Element> nonZeroElements = toAdd.iterateNonZero(); while (nonZeroElements.hasNext()) { Vector.Element nonZeroElement = nonZeroElements.next(); dots.setQuick( nonZeroElement.index(), dots.getQuick(nonZeroElement.index()) + nonZeroElement.get()); } } Vector similarities = dots.like(); double normA = norms.getQuick(row.get()); Iterator<Vector.Element> dotsWith = dots.iterateNonZero(); while (dotsWith.hasNext()) { Vector.Element b = dotsWith.next(); double similarityValue = similarity.similarity(b.get(), normA, norms.getQuick(b.index()), numberOfColumns); if (similarityValue >= treshold) { similarities.set(b.index(), similarityValue); } } if (excludeSelfSimilarity) { similarities.setQuick(row.get(), 0); } ctx.write(row, new VectorWritable(similarities)); }
@Override protected void map(IntWritable row, VectorWritable vectorWritable, Context ctx) throws IOException, InterruptedException { Vector rowVector = similarity.normalize(vectorWritable.get()); int numNonZeroEntries = 0; double maxValue = Double.MIN_VALUE; Iterator<Vector.Element> nonZeroElements = rowVector.iterateNonZero(); while (nonZeroElements.hasNext()) { Vector.Element element = nonZeroElements.next(); RandomAccessSparseVector partialColumnVector = new RandomAccessSparseVector(Integer.MAX_VALUE); partialColumnVector.setQuick(row.get(), element.get()); ctx.write(new IntWritable(element.index()), new VectorWritable(partialColumnVector)); numNonZeroEntries++; if (maxValue < element.get()) { maxValue = element.get(); } } if (threshold != NO_THRESHOLD) { nonZeroEntries.setQuick(row.get(), numNonZeroEntries); maxValues.setQuick(row.get(), maxValue); } norms.setQuick(row.get(), similarity.norm(rowVector)); ctx.getCounter(Counters.ROWS).increment(1); }
public static IntWritable minMsg(Iterable<IntWritable> messages) { int min = Integer.MAX_VALUE; for (IntWritable msg : messages) { min = Math.min(min, msg.get()); } return new IntWritable(min); }
public static void main(String[] args) throws IOException { // TODO Auto-generated method stub String uri = args[0]; Configuration conf = new Configuration(); Path path = new Path(uri); FileSystem fs = FileSystem.get(URI.create(uri), conf); IntWritable key = new IntWritable(); Text value = new Text(); SequenceFile.Writer writer = null; try { writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass()); for (int i = 0; i < 100; i++) { key.set(100 - i); value.set(DATA[i % DATA.length]); writer.append(key, value); } } finally { IOUtils.closeStream(writer); } }
public void run( RecordReader<IntWritable, WikipediaPage> input, OutputCollector<IntWritable, Text> output, Reporter reporter) throws IOException { IntWritable key = new IntWritable(); WikipediaPage value = new WikipediaPage(); long pos = -1; long prevPos = -1; int prevDocno = 0; pos = input.getPos(); while (input.next(key, value)) { if (prevPos != -1 && prevPos != pos) { LOG.info( "- beginning of block at " + prevPos + ", docno:" + prevDocno + ", file:" + fileno); keyOut.set(prevDocno); valOut.set(prevPos + "\t" + fileno); output.collect(keyOut, valOut); reporter.incrCounter(Blocks.Total, 1); } prevPos = pos; pos = input.getPos(); prevDocno = key.get(); } }
public void map( LongWritable key, Text value, OutputCollector<IntWritable, HITSNode> output, Reporter reporter) throws IOException { ArrayListOfIntsWritable links = new ArrayListOfIntsWritable(); String line = ((Text) value).toString(); StringTokenizer itr = new StringTokenizer(line); if (itr.hasMoreTokens()) { int curr = Integer.parseInt(itr.nextToken()); if (stopList.contains(curr)) { return; } valOut.setAdjacencyList(links); valOut.setHARank((float) 1.0); valOut.setType(HITSNode.TYPE_AUTH_COMPLETE); } while (itr.hasMoreTokens()) { keyOut.set(Integer.parseInt(itr.nextToken())); valOut.setNodeId(keyOut.get()); // System.out.println(keyOut.toString() + ", " + // valOut.toString()); if (!(stopList.contains(keyOut.get()))) { output.collect(keyOut, valOut); } } // emit mentioned mentioner -> mentioned (mentioners) in links // emit mentioner mentioned -> mentioner (mentions) outlinks // emit mentioned a // emit mentioner 1 }
@Before public void setUp() throws Exception { // create local Pig server pigServer = UnitTestUtil.makePigServer(); // create temp SequenceFile File tempFile = File.createTempFile("test", ".txt"); tempFilename = tempFile.getAbsolutePath(); Path path = new Path("file:///" + tempFilename); Configuration conf = new Configuration(); FileSystem fs = path.getFileSystem(conf); IntWritable key = new IntWritable(); Text value = new Text(); SequenceFile.Writer writer = null; try { writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass()); for (int i = 0; i < DATA.length; ++i) { key.set(i); value.set(DATA[i]); writer.append(key, value); } } finally { IOUtils.closeStream(writer); } }
public static double getAsConstDouble(@Nonnull final ObjectInspector numberOI) throws UDFArgumentException { final String typeName = numberOI.getTypeName(); if (DOUBLE_TYPE_NAME.equals(typeName)) { DoubleWritable v = getConstValue(numberOI); return v.get(); } else if (FLOAT_TYPE_NAME.equals(typeName)) { FloatWritable v = getConstValue(numberOI); return v.get(); } else if (INT_TYPE_NAME.equals(typeName)) { IntWritable v = getConstValue(numberOI); return v.get(); } else if (BIGINT_TYPE_NAME.equals(typeName)) { LongWritable v = getConstValue(numberOI); return v.get(); } else if (SMALLINT_TYPE_NAME.equals(typeName)) { ShortWritable v = getConstValue(numberOI); return v.get(); } else if (TINYINT_TYPE_NAME.equals(typeName)) { ByteWritable v = getConstValue(numberOI); return v.get(); } throw new UDFArgumentException( "Unexpected argument type to cast as double: " + TypeInfoUtils.getTypeInfoFromObjectInspector(numberOI)); }
/** * @param state The final LanczosState to be serialized * @param outputPath The path (relative to the current Configuration's FileSystem) to save the * output to. */ public void serializeOutput(LanczosState state, Path outputPath) throws IOException { int numEigenVectors = state.getIterationNumber(); log.info("Persisting {} eigenVectors and eigenValues to: {}", numEigenVectors, outputPath); Configuration conf = getConf() != null ? getConf() : new Configuration(); FileSystem fs = FileSystem.get(outputPath.toUri(), conf); SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, outputPath, IntWritable.class, VectorWritable.class); try { IntWritable iw = new IntWritable(); for (int i = 0; i < numEigenVectors; i++) { // Persist eigenvectors sorted by eigenvalues in descending order\ NamedVector v = new NamedVector( state.getRightSingularVector(numEigenVectors - 1 - i), "eigenVector" + i + ", eigenvalue = " + state.getSingularValue(numEigenVectors - 1 - i)); Writable vw = new VectorWritable(v); iw.set(i); seqWriter.append(iw, vw); } } finally { Closeables.close(seqWriter, false); } }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; boolean filtered = true; String keypair = key.toString(); String[] keys = keypair.split(" "); if (keys.length == 1) { filtered = false; } else { if (keys[0].equals("*") || keys[1].equals("*")) { filtered = false; } } if (!filtered) { for (IntWritable val : values) { sum += val.get(); } context.write(key, new IntWritable(sum)); return; } for (IntWritable val : values) { if (val.get() == -1) { filtered = false; continue; } sum += val.get(); } // filter non-needed events if (filtered) return; context.write(key, new IntWritable(sum)); }
@Override public int compareTo(KeyPair otherPair) { // TODO Auto-generated method stub int c = ArivalDelay.compareTo(otherPair.ArivalDelay); if (c != 0) return c; else return Flightnumber.compareTo(otherPair.Flightnumber); }
@Override public boolean equals(Object o) { if (o instanceof PairIntWritable) { PairIntWritable tp = (PairIntWritable) o; return first.equals(tp.first) && second.equals(tp.second); } return false; }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int maxValues = Integer.MIN_VALUE; for (IntWritable value : values) { maxValues = Math.max(maxValues, value.get()); } context.write(key, new IntWritable(maxValues)); }
public static int getConstInt(@Nonnull final ObjectInspector oi) throws UDFArgumentException { if (!isIntOI(oi)) { throw new UDFArgumentException( "argument must be a Int value: " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); } IntWritable v = getConstValue(oi); return v.get(); }
@Override public int compareTo(PairIntWritable tp) { int cmp = first.compareTo(tp.first); if (cmp != 0) { return cmp; } return second.compareTo(tp.second); }
public void reduce(BytesWritable key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } context.write(key, new IntWritable(sum)); }
public void reduce(Text key, Iterable<IntWritable> results, Context context) throws IOException, InterruptedException { int total = 0; for (IntWritable cost : results) { total += cost.get(); } context.write(key, new IntWritable(total)); }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int i = 0; for (IntWritable val : values) { i += val.get(); } context.write(key, new IntWritable(i)); }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } countMap.put(new Text(key), new IntWritable(sum)); }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable value : values) { sum += value.get(); } reducedValue.set(sum); context.write(key, new IntWritable(sum)); }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int totalFreq = 0; for (IntWritable value : values) { totalFreq = totalFreq + value.get(); } context.write(key, new IntWritable(totalFreq)); }