@Override
  public void compute(
      Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages)
      throws IOException {
    int currentLength = vertex.getValue().get();
    // First superstep is special, because we can simply look at the neighbors
    if (getSuperstep() == 0) {
      if (isSource(vertex)) {
        vertex.setValue(new IntWritable(0));
        sendMessageToAllEdges(vertex, vertex.getValue());
      } else {
        vertex.setValue(new IntWritable(Integer.MAX_VALUE));
      }
      vertex.voteToHalt();
      return;
    }

    boolean changed = false;
    // did we get a smaller length ?
    for (IntWritable message : messages) {
      int candidateLength = message.get() + 1;
      if (candidateLength < currentLength) {
        currentLength = candidateLength;
        changed = true;
      }
    }

    // propagate new length to the neighbors
    if (changed) {
      vertex.setValue(new IntWritable(currentLength));
      sendMessageToAllEdges(vertex, vertex.getValue());
    }
    vertex.voteToHalt();
  }
Пример #2
0
 @Override
 protected void reduce(IntWritable key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   for (IntWritable i : values) {
     set.add(i.get());
   }
 }
Пример #3
0
      public void reduce(
          BytesWritable key,
          Iterator<IntWritable> values,
          OutputCollector<BytesWritable, IntWritable> output,
          Reporter reporter)
          throws IOException {
        int ones = 0;
        int twos = 0;
        while (values.hasNext()) {
          IntWritable count = values.next();
          if (count.equals(sortInput)) {
            ++ones;
          } else if (count.equals(sortOutput)) {
            ++twos;
          } else {
            throw new IOException(
                "Invalid 'value' of " + count.get() + " for (key,value): " + key.toString());
          }
        }

        // Check to ensure there are equal no. of ones and twos
        if (ones != twos) {
          throw new IOException(
              "Illegal ('one', 'two'): ("
                  + ones
                  + ", "
                  + twos
                  + ") for (key, value): "
                  + key.toString());
        }
      }
Пример #4
0
  @Override
  public void compute(Iterable<IntWritable> messages) {
    if (getId().equals(new LongWritable(2))) {
      sendMessage(new LongWritable(1), new IntWritable(101));
      sendMessage(new LongWritable(1), new IntWritable(102));
      sendMessage(new LongWritable(1), new IntWritable(103));
    }
    if (!getId().equals(new LongWritable(1))) {
      voteToHalt();
    } else {
      // Check the messages
      int sum = 0;
      int num = 0;
      for (IntWritable message : messages) {
        sum += message.get();
        num++;
      }
      LOG.info(
          "TestCombinerVertex: Received a sum of "
              + sum
              + " (should have 306 with a single message value)");

      if (num == 1 && sum == 306) {
        voteToHalt();
      }
    }
    if (getSuperstep() > 3) {
      throw new IllegalStateException(
          "TestCombinerVertex: Vertex 1 failed to receive " + "messages in time");
    }
  }
Пример #5
0
 @Override
 protected void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable value : values) sum += value.get();
   context.write(key, new IntWritable(sum));
 }
  @Override
  public void compute(
      Vertex<IntWritable, IntWritable, NullWritable> vertex, Iterable<IntWritable> messages)
      throws IOException {

    if (0 == getSuperstep()) {
      vertex.setValue(new IntWritable(Integer.MAX_VALUE));
    }

    Integer minDist = isSource(vertex) ? 0 : Integer.MAX_VALUE;
    for (IntWritable msg : messages) {
      if (msg.get() < minDist) {
        minDist = msg.get();
      }
    }

    if (minDist < vertex.getValue().get()) {
      vertex.setValue(new IntWritable(minDist));

      for (Edge<IntWritable, NullWritable> edge : vertex.getEdges()) {
        final IntWritable neighbor = edge.getTargetVertexId();
        final Integer distance = minDist + 1;

        sendMessage(neighbor, new IntWritable(distance));
      }
    }

    vertex.voteToHalt();
  }
  @Override
  public void reduce(Text key, Iterable<IntWritable> values, Context context)
      throws IOException, InterruptedException {

    int ratingsCount = 0;

    /*
     * For each movieId:
     */
    // Define a counter for bad records.
    for (IntWritable value : values) {

      /*
       * Count the movie ratings.
       */
      ratingsCount += value.get();
    }

    /*
     * Reducer output is the userID and number of ratings.
     */
    context.getCounter(UserIdCounter.NUM_USERS).increment(1);
    rValue.set(ratingsCount);
    context.write(key, rValue);
  }
    @Override
    protected void map(IntWritable row, VectorWritable similaritiesWritable, Context ctx)
        throws IOException, InterruptedException {
      Vector similarities = similaritiesWritable.get();
      // For performance, the creation of transposedPartial is moved out of the while loop and it is
      // reused inside
      Vector transposedPartial = new RandomAccessSparseVector(similarities.size(), 1);
      TopElementsQueue topKQueue = new TopElementsQueue(maxSimilaritiesPerRow);
      Iterator<Vector.Element> nonZeroElements = similarities.iterateNonZero();
      while (nonZeroElements.hasNext()) {
        Vector.Element nonZeroElement = nonZeroElements.next();

        MutableElement top = topKQueue.top();
        double candidateValue = nonZeroElement.get();
        if (candidateValue > top.get()) {
          top.setIndex(nonZeroElement.index());
          top.set(candidateValue);
          topKQueue.updateTop();
        }

        transposedPartial.setQuick(row.get(), candidateValue);
        ctx.write(new IntWritable(nonZeroElement.index()), new VectorWritable(transposedPartial));
        transposedPartial.setQuick(row.get(), 0.0);
      }
      Vector topKSimilarities =
          new RandomAccessSparseVector(similarities.size(), maxSimilaritiesPerRow);
      for (Vector.Element topKSimilarity : topKQueue.getTopElements()) {
        topKSimilarities.setQuick(topKSimilarity.index(), topKSimilarity.get());
      }
      ctx.write(row, new VectorWritable(topKSimilarities));
    }
Пример #9
0
 @Override
 public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) {
   size.value = SIZE;
   IntWritable writable = new IntWritable();
   writable.set(typeConverter.convertTo(Integer.class, value));
   return writable;
 }
    @Override
    protected void reduce(IntWritable row, Iterable<VectorWritable> partialDots, Context ctx)
        throws IOException, InterruptedException {
      Iterator<VectorWritable> partialDotsIterator = partialDots.iterator();
      Vector dots = partialDotsIterator.next().get();
      while (partialDotsIterator.hasNext()) {
        Vector toAdd = partialDotsIterator.next().get();
        Iterator<Vector.Element> nonZeroElements = toAdd.iterateNonZero();
        while (nonZeroElements.hasNext()) {
          Vector.Element nonZeroElement = nonZeroElements.next();
          dots.setQuick(
              nonZeroElement.index(), dots.getQuick(nonZeroElement.index()) + nonZeroElement.get());
        }
      }

      Vector similarities = dots.like();
      double normA = norms.getQuick(row.get());
      Iterator<Vector.Element> dotsWith = dots.iterateNonZero();
      while (dotsWith.hasNext()) {
        Vector.Element b = dotsWith.next();
        double similarityValue =
            similarity.similarity(b.get(), normA, norms.getQuick(b.index()), numberOfColumns);
        if (similarityValue >= treshold) {
          similarities.set(b.index(), similarityValue);
        }
      }
      if (excludeSelfSimilarity) {
        similarities.setQuick(row.get(), 0);
      }
      ctx.write(row, new VectorWritable(similarities));
    }
    @Override
    protected void map(IntWritable row, VectorWritable vectorWritable, Context ctx)
        throws IOException, InterruptedException {

      Vector rowVector = similarity.normalize(vectorWritable.get());

      int numNonZeroEntries = 0;
      double maxValue = Double.MIN_VALUE;

      Iterator<Vector.Element> nonZeroElements = rowVector.iterateNonZero();
      while (nonZeroElements.hasNext()) {
        Vector.Element element = nonZeroElements.next();
        RandomAccessSparseVector partialColumnVector =
            new RandomAccessSparseVector(Integer.MAX_VALUE);
        partialColumnVector.setQuick(row.get(), element.get());
        ctx.write(new IntWritable(element.index()), new VectorWritable(partialColumnVector));

        numNonZeroEntries++;
        if (maxValue < element.get()) {
          maxValue = element.get();
        }
      }

      if (threshold != NO_THRESHOLD) {
        nonZeroEntries.setQuick(row.get(), numNonZeroEntries);
        maxValues.setQuick(row.get(), maxValue);
      }
      norms.setQuick(row.get(), similarity.norm(rowVector));

      ctx.getCounter(Counters.ROWS).increment(1);
    }
Пример #12
0
 public static IntWritable minMsg(Iterable<IntWritable> messages) {
   int min = Integer.MAX_VALUE;
   for (IntWritable msg : messages) {
     min = Math.min(min, msg.get());
   }
   return new IntWritable(min);
 }
Пример #13
0
  public static void main(String[] args) throws IOException {
    // TODO Auto-generated method stub
    String uri = args[0];

    Configuration conf = new Configuration();

    Path path = new Path(uri);

    FileSystem fs = FileSystem.get(URI.create(uri), conf);

    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {
      writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());

      for (int i = 0; i < 100; i++) {
        key.set(100 - i);
        value.set(DATA[i % DATA.length]);

        writer.append(key, value);
      }
    } finally {
      IOUtils.closeStream(writer);
    }
  }
Пример #14
0
    public void run(
        RecordReader<IntWritable, WikipediaPage> input,
        OutputCollector<IntWritable, Text> output,
        Reporter reporter)
        throws IOException {
      IntWritable key = new IntWritable();
      WikipediaPage value = new WikipediaPage();

      long pos = -1;
      long prevPos = -1;

      int prevDocno = 0;

      pos = input.getPos();
      while (input.next(key, value)) {
        if (prevPos != -1 && prevPos != pos) {
          LOG.info(
              "- beginning of block at " + prevPos + ", docno:" + prevDocno + ", file:" + fileno);
          keyOut.set(prevDocno);
          valOut.set(prevPos + "\t" + fileno);
          output.collect(keyOut, valOut);
          reporter.incrCounter(Blocks.Total, 1);
        }

        prevPos = pos;
        pos = input.getPos();
        prevDocno = key.get();
      }
    }
Пример #15
0
    public void map(
        LongWritable key,
        Text value,
        OutputCollector<IntWritable, HITSNode> output,
        Reporter reporter)
        throws IOException {

      ArrayListOfIntsWritable links = new ArrayListOfIntsWritable();
      String line = ((Text) value).toString();
      StringTokenizer itr = new StringTokenizer(line);
      if (itr.hasMoreTokens()) {
        int curr = Integer.parseInt(itr.nextToken());
        if (stopList.contains(curr)) {
          return;
        }
        valOut.setAdjacencyList(links);
        valOut.setHARank((float) 1.0);
        valOut.setType(HITSNode.TYPE_AUTH_COMPLETE);
      }
      while (itr.hasMoreTokens()) {
        keyOut.set(Integer.parseInt(itr.nextToken()));
        valOut.setNodeId(keyOut.get());
        // System.out.println(keyOut.toString() + ", " +
        // valOut.toString());
        if (!(stopList.contains(keyOut.get()))) {
          output.collect(keyOut, valOut);
        }
      }
      // emit mentioned mentioner -> mentioned (mentioners) in links
      // emit mentioner mentioned -> mentioner (mentions) outlinks
      // emit mentioned a
      // emit mentioner 1
    }
  @Before
  public void setUp() throws Exception {
    // create local Pig server
    pigServer = UnitTestUtil.makePigServer();

    // create temp SequenceFile
    File tempFile = File.createTempFile("test", ".txt");
    tempFilename = tempFile.getAbsolutePath();
    Path path = new Path("file:///" + tempFilename);
    Configuration conf = new Configuration();
    FileSystem fs = path.getFileSystem(conf);
    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {
      writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
      for (int i = 0; i < DATA.length; ++i) {
        key.set(i);
        value.set(DATA[i]);
        writer.append(key, value);
      }
    } finally {
      IOUtils.closeStream(writer);
    }
  }
Пример #17
0
 public static double getAsConstDouble(@Nonnull final ObjectInspector numberOI)
     throws UDFArgumentException {
   final String typeName = numberOI.getTypeName();
   if (DOUBLE_TYPE_NAME.equals(typeName)) {
     DoubleWritable v = getConstValue(numberOI);
     return v.get();
   } else if (FLOAT_TYPE_NAME.equals(typeName)) {
     FloatWritable v = getConstValue(numberOI);
     return v.get();
   } else if (INT_TYPE_NAME.equals(typeName)) {
     IntWritable v = getConstValue(numberOI);
     return v.get();
   } else if (BIGINT_TYPE_NAME.equals(typeName)) {
     LongWritable v = getConstValue(numberOI);
     return v.get();
   } else if (SMALLINT_TYPE_NAME.equals(typeName)) {
     ShortWritable v = getConstValue(numberOI);
     return v.get();
   } else if (TINYINT_TYPE_NAME.equals(typeName)) {
     ByteWritable v = getConstValue(numberOI);
     return v.get();
   }
   throw new UDFArgumentException(
       "Unexpected argument type to cast as double: "
           + TypeInfoUtils.getTypeInfoFromObjectInspector(numberOI));
 }
Пример #18
0
 /**
  * @param state The final LanczosState to be serialized
  * @param outputPath The path (relative to the current Configuration's FileSystem) to save the
  *     output to.
  */
 public void serializeOutput(LanczosState state, Path outputPath) throws IOException {
   int numEigenVectors = state.getIterationNumber();
   log.info("Persisting {} eigenVectors and eigenValues to: {}", numEigenVectors, outputPath);
   Configuration conf = getConf() != null ? getConf() : new Configuration();
   FileSystem fs = FileSystem.get(outputPath.toUri(), conf);
   SequenceFile.Writer seqWriter =
       new SequenceFile.Writer(fs, conf, outputPath, IntWritable.class, VectorWritable.class);
   try {
     IntWritable iw = new IntWritable();
     for (int i = 0; i < numEigenVectors; i++) {
       // Persist eigenvectors sorted by eigenvalues in descending order\
       NamedVector v =
           new NamedVector(
               state.getRightSingularVector(numEigenVectors - 1 - i),
               "eigenVector"
                   + i
                   + ", eigenvalue = "
                   + state.getSingularValue(numEigenVectors - 1 - i));
       Writable vw = new VectorWritable(v);
       iw.set(i);
       seqWriter.append(iw, vw);
     }
   } finally {
     Closeables.close(seqWriter, false);
   }
 }
Пример #19
0
    public void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws IOException, InterruptedException {
      int sum = 0;
      boolean filtered = true;
      String keypair = key.toString();
      String[] keys = keypair.split(" ");
      if (keys.length == 1) {
        filtered = false;
      } else {
        if (keys[0].equals("*") || keys[1].equals("*")) {
          filtered = false;
        }
      }

      if (!filtered) {
        for (IntWritable val : values) {
          sum += val.get();
        }
        context.write(key, new IntWritable(sum));
        return;
      }

      for (IntWritable val : values) {
        if (val.get() == -1) {
          filtered = false;
          continue;
        }
        sum += val.get();
      }
      // filter non-needed events
      if (filtered) return;
      context.write(key, new IntWritable(sum));
    }
Пример #20
0
 @Override
 public int compareTo(KeyPair otherPair) {
   // TODO Auto-generated method stub
   int c = ArivalDelay.compareTo(otherPair.ArivalDelay);
   if (c != 0) return c;
   else return Flightnumber.compareTo(otherPair.Flightnumber);
 }
Пример #21
0
 @Override
 public boolean equals(Object o) {
   if (o instanceof PairIntWritable) {
     PairIntWritable tp = (PairIntWritable) o;
     return first.equals(tp.first) && second.equals(tp.second);
   }
   return false;
 }
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int maxValues = Integer.MIN_VALUE;
   for (IntWritable value : values) {
     maxValues = Math.max(maxValues, value.get());
   }
   context.write(key, new IntWritable(maxValues));
 }
Пример #23
0
 public static int getConstInt(@Nonnull final ObjectInspector oi) throws UDFArgumentException {
   if (!isIntOI(oi)) {
     throw new UDFArgumentException(
         "argument must be a Int value: " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi));
   }
   IntWritable v = getConstValue(oi);
   return v.get();
 }
Пример #24
0
 @Override
 public int compareTo(PairIntWritable tp) {
   int cmp = first.compareTo(tp.first);
   if (cmp != 0) {
     return cmp;
   }
   return second.compareTo(tp.second);
 }
Пример #25
0
 public void reduce(BytesWritable key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
     sum += val.get();
   }
   context.write(key, new IntWritable(sum));
 }
Пример #26
0
 public void reduce(Text key, Iterable<IntWritable> results, Context context)
     throws IOException, InterruptedException {
   int total = 0;
   for (IntWritable cost : results) {
     total += cost.get();
   }
   context.write(key, new IntWritable(total));
 }
Пример #27
0
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int i = 0;
   for (IntWritable val : values) {
     i += val.get();
   }
   context.write(key, new IntWritable(i));
 }
Пример #28
0
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
     sum += val.get();
   }
   countMap.put(new Text(key), new IntWritable(sum));
 }
Пример #29
0
 public void reduce(Text key, Iterable<IntWritable> values, Context context)
     throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable value : values) {
     sum += value.get();
   }
   reducedValue.set(sum);
   context.write(key, new IntWritable(sum));
 }
Пример #30
0
    public void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws IOException, InterruptedException {

      int totalFreq = 0;
      for (IntWritable value : values) {
        totalFreq = totalFreq + value.get();
      }
      context.write(key, new IntWritable(totalFreq));
    }