Пример #1
0
 @Override
 public void reduce(
     GenericKey key,
     Iterator<GenericValue> values,
     OutputCollector<GenericKey, GenericValue> output,
     Reporter reporter)
     throws IOException {
   if (key.getSecondary() < Preprocesser.MINIMUM_ID) { // vector
     output.collect(key, values.next());
     if (values.hasNext()) assert false : "Vectors should not get grouped by combiner: " + key;
   } else { // addend
     reporter.progress();
     int counter = 0;
     float sim = 0;
     HalfPair hp = null;
     while (values.hasNext()) {
       hp = (HalfPair) values.next().get();
       sim += hp.getSimilarity();
       if (counter++ % REPORTER_INTERVAL == 0) reporter.progress();
     }
     if (hp != null) {
       payload.set(hp.getID(), sim);
       outValue.set(payload);
       output.collect(key, outValue);
     } else {
       assert false : "There is nothing to combine!";
     }
   }
 }
Пример #2
0
 @Override
 public void map(
     IntWritable vectorID,
     VectorComponentArrayWritable value,
     OutputCollector<GenericKey, GenericValue> output,
     Reporter reporter)
     throws IOException {
   // vectors sort before pairs using a secondary key < MINIMUM_ID
   for (int i = 1; i <= nstripes; i++) {
     outKey.set(vectorID.get(), Preprocesser.MINIMUM_ID - i);
     outValue.set(value);
     output.collect(outKey, outValue);
   }
 }
Пример #3
0
 @Override
 public void map(
     LongWritable key,
     IndexItemArrayWritable value,
     OutputCollector<GenericKey, GenericValue> output,
     Reporter reporter)
     throws IOException {
   IndexItem[] postingList = value.toIndexItemArray();
   for (int i = 1; i < postingList.length; i++) {
     for (int j = 0; j < i; j++) {
       IndexItem x = postingList[i];
       IndexItem y = postingList[j];
       // |y| >= t / maxweight(x) && |x| >= t / maxweight(y)
       if (compare(x.vectorLength(), Math.ceil(threshold / y.vectorMaxWeight())) >= 0
           && compare(y.vectorLength(), Math.ceil(threshold / x.vectorMaxWeight())) >= 0
           // tight upper bound on similarity score
           && compare(
                   min(x.vectorMaxWeight() * y.vectorSum(), y.vectorMaxWeight() * x.vectorSum()),
                   threshold)
               >= 0) {
         // positional filter
         // && compare(
         // min(x.positionalMaxWeight() * y.positionalSum(),
         // y.positionalMaxWeight() * x.positionalSum())
         // + x.getWeight() * y.getWeight(), threshold) >= 0)
         if (j % REPORTER_INTERVAL == 0) reporter.progress();
         int lpv = IndexItem.getLeastPrunedVectorID(x, y);
         int mpv = IndexItem.getMostPrunedVectorID(x, y);
         float psim = (float) (x.getWeight() * y.getWeight());
         outKey.set(lpv, mpv);
         payload.set(mpv, psim);
         outValue.set(payload);
         output.collect(outKey, outValue);
         reporter.incrCounter(APS.ADDEND, 1);
       }
     }
   }
 }