コード例 #1
0
 /** Explain the score we computed for doc */
 @Override
 public Explanation explain(LeafReaderContext context, int doc) throws IOException {
   boolean match = false;
   float max = 0.0f, sum = 0.0f;
   List<Explanation> subs = new ArrayList<>();
   for (Weight wt : weights) {
     Explanation e = wt.explain(context, doc);
     if (e.isMatch()) {
       match = true;
       subs.add(e);
       sum += e.getValue();
       max = Math.max(max, e.getValue());
     }
   }
   if (match) {
     final float score = max + (sum - max) * tieBreakerMultiplier;
     final String desc =
         tieBreakerMultiplier == 0.0f
             ? "max of:"
             : "max plus " + tieBreakerMultiplier + " times others of:";
     return Explanation.match(score, desc, subs);
   } else {
     return Explanation.noMatch("No matching clause");
   }
 }
 /** Apply the computed normalization factor to our subqueries */
 @Override
 public void normalize(float norm, float topLevelBoost) {
   topLevelBoost *= getBoost(); // Incorporate our boost
   for (Weight wt : weights) {
     wt.normalize(norm, topLevelBoost);
   }
 }
コード例 #3
0
    @Override
    public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
      if (used.compareAndSet(false, true)) {
        policy.onUse(getQuery());
      }
      DocIdSet docIdSet = get(in.getQuery(), context);
      if (docIdSet == null) {
        if (shouldCache(context)) {
          docIdSet = cache(context);
          putIfAbsent(in.getQuery(), context, docIdSet);
        } else {
          return in.bulkScorer(context);
        }
      }

      assert docIdSet != null;
      if (docIdSet == DocIdSet.EMPTY) {
        return null;
      }
      final DocIdSetIterator disi = docIdSet.iterator();
      if (disi == null) {
        return null;
      }

      return new DefaultBulkScorer(new ConstantScoreScorer(this, 0f, disi));
    }
コード例 #4
0
 /**
  * Compute the sub of squared weights of us applied to our subqueries. Used for normalization.
  */
 @Override
 public float getValueForNormalization() throws IOException {
   float max = 0.0f, sum = 0.0f;
   for (Weight currentWeight : weights) {
     float sub = currentWeight.getValueForNormalization();
     sum += sub;
     max = Math.max(max, sub);
   }
   return (((sum - max) * tieBreakerMultiplier * tieBreakerMultiplier) + max);
 }
コード例 #5
0
  public void testSkipToFirsttimeHit() throws IOException {
    final DisjunctionMaxQuery dq = new DisjunctionMaxQuery(0.0f);
    dq.add(tq("dek", "albino"));
    dq.add(tq("dek", "DOES_NOT_EXIST"));

    QueryUtils.check(dq, s);

    final Weight dw = dq.weight(s);
    final Scorer ds = dw.scorer(s.getIndexReader(), true, false);
    assertTrue("firsttime skipTo found no match", ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS);
    assertEquals("found wrong docid", "d4", r.document(ds.docID()).get("id"));
  }
コード例 #6
0
  public void testSkipToFirsttimeMiss() throws IOException {
    final DisjunctionMaxQuery dq = new DisjunctionMaxQuery(0.0f);
    dq.add(tq("id", "d1"));
    dq.add(tq("dek", "DOES_NOT_EXIST"));

    QueryUtils.check(dq, s);

    final Weight dw = dq.weight(s);
    final Scorer ds = dw.scorer(s.getIndexReader(), true, false);
    final boolean skipOk = ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS;
    if (skipOk) {
      fail("firsttime skipTo found a match? ... " + r.document(ds.docID()).get("id"));
    }
  }
コード例 #7
0
 @Override
 public void normalize(float norm, float topLevelBoost) {
   this.queryNorm = norm * topLevelBoost;
   queryWeight *= this.queryNorm;
   // we normalize the inner weight, but ignore it (just to initialize everything)
   if (innerWeight != null) innerWeight.normalize(norm, topLevelBoost);
 }
コード例 #8
0
    @Override
    public Scorer scorer(
        AtomicReaderContext context,
        boolean scoreDocsInOrder,
        boolean topScorer,
        final Bits acceptDocs)
        throws IOException {
      final DocIdSetIterator disi;
      if (filter != null) {
        assert query == null;
        final DocIdSet dis = filter.getDocIdSet(context, acceptDocs);
        if (dis == null) {
          return null;
        }
        disi = dis.iterator();
      } else {
        assert query != null && innerWeight != null;
        disi = innerWeight.scorer(context, scoreDocsInOrder, topScorer, acceptDocs);
      }

      if (disi == null) {
        return null;
      }
      return new ConstantScorer(disi, this, queryWeight);
    }
コード例 #9
0
  // inherit javadoc
  public void search(Weight weight, Filter filter, final HitCollector results) throws IOException {

    Scorer scorer = weight.scorer(reader);
    if (scorer == null) return;

    if (filter == null) {
      scorer.score(results);
      return;
    }
    DocIdSetIterator filterDocIdIterator =
        filter.getDocIdSet(reader).iterator(); // CHECKME: use ConjunctionScorer here?

    boolean more = filterDocIdIterator.next() && scorer.skipTo(filterDocIdIterator.doc());

    while (more) {
      int filterDocId = filterDocIdIterator.doc();
      if (filterDocId > scorer.doc() && !scorer.skipTo(filterDocId)) {
        more = false;
      } else {
        int scorerDocId = scorer.doc();
        if (scorerDocId == filterDocId) { // permitted by filter
          results.collect(scorerDocId, scorer.score());
          more = filterDocIdIterator.next();
        } else {
          more = filterDocIdIterator.skipTo(scorerDocId);
        }
      }
    }
  }
コード例 #10
0
 @Override
 public float getValueForNormalization() throws IOException {
   // we calculate sumOfSquaredWeights of the inner weight, but ignore it (just to initialize
   // everything)
   if (innerWeight != null) innerWeight.getValueForNormalization();
   queryWeight = getBoost();
   return queryWeight * queryWeight;
 }
コード例 #11
0
 private DocIdSet cache(LeafReaderContext context) throws IOException {
   final BulkScorer scorer = in.bulkScorer(context);
   if (scorer == null) {
     return DocIdSet.EMPTY;
   } else {
     return cacheImpl(scorer, context.reader().maxDoc());
   }
 }
コード例 #12
0
 @Override
 public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer)
     throws IOException {
   Scorer subQueryScorer = qWeight.scorer(reader, true, false);
   if (subQueryScorer == null) {
     return null;
   }
   return new BoostedQuery.CustomScorer(
       getSimilarity(searcher), searcher, reader, this, subQueryScorer, boostVal);
 }
コード例 #13
0
ファイル: Weight.java プロジェクト: 578608614/quiship
 public synchronized boolean equals(java.lang.Object obj) {
   if (!(obj instanceof Weight)) return false;
   Weight other = (Weight) obj;
   if (obj == null) return false;
   if (this == obj) return true;
   if (__equalsCalc != null) {
     return (__equalsCalc == obj);
   }
   __equalsCalc = obj;
   boolean _equals;
   _equals =
       true
           && ((this.units == null && other.getUnits() == null)
               || (this.units != null && this.units.equals(other.getUnits())))
           && ((this.value == null && other.getValue() == null)
               || (this.value != null && this.value.equals(other.getValue())));
   __equalsCalc = null;
   return _equals;
 }
コード例 #14
0
 /** Create the scorer used to score our associated DisjunctionMaxQuery */
 @Override
 public Scorer scorer(LeafReaderContext context) throws IOException {
   List<Scorer> scorers = new ArrayList<>();
   for (Weight w : weights) {
     // we will advance() subscorers
     Scorer subScorer = w.scorer(context);
     if (subScorer != null) {
       scorers.add(subScorer);
     }
   }
   if (scorers.isEmpty()) {
     // no sub-scorers had any documents
     return null;
   } else if (scorers.size() == 1) {
     // only one sub-scorer in this segment
     return scorers.get(0);
   } else {
     return new DisjunctionMaxScorer(this, tieBreakerMultiplier, scorers, needsScores);
   }
 }
コード例 #15
0
 /** Explain the score we computed for doc */
 @Override
 public Explanation explain(LeafReaderContext context, int doc) throws IOException {
   if (disjuncts.size() == 1) return weights.get(0).explain(context, doc);
   ComplexExplanation result = new ComplexExplanation();
   float max = 0.0f, sum = 0.0f;
   result.setDescription(
       tieBreakerMultiplier == 0.0f
           ? "max of:"
           : "max plus " + tieBreakerMultiplier + " times others of:");
   for (Weight wt : weights) {
     Explanation e = wt.explain(context, doc);
     if (e.isMatch()) {
       result.setMatch(Boolean.TRUE);
       result.addDetail(e);
       sum += e.getValue();
       max = Math.max(max, e.getValue());
     }
   }
   result.setValue(max + (sum - max) * tieBreakerMultiplier);
   return result;
 }
コード例 #16
0
  public static Weight[] calculateWeights(List list, String valueRef)
      throws NoSuchFieldException, SecurityException, IllegalArgumentException,
          IllegalAccessException {
    Weight weight;
    int sum = 0;
    int size = list.size();
    Weight[] resultList = new Weight[size];

    for (int i = 0; i < size; i++) {
      Object item = list.get(i);
      Class cls = item.getClass();
      Field field = cls.getDeclaredField(valueRef);

      int value = field.getInt(item);
      sum += value;

      weight = new Weight();
      weight.data = item;
      weight.value = value;
      resultList[i] = weight;
    }

    double inc = 0;

    for (int j = 0; j < size; j++) {
      weight = resultList[j];
      weight.proportion = weight.value / sum;
      weight.lower = inc;
      inc += weight.proportion;
      weight.upper = inc;
    }

    return resultList;
  }
コード例 #17
0
 @Override
 public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer)
     throws IOException {
   final DocIdSetIterator disi;
   if (filter != null) {
     assert query == null;
     final DocIdSet dis = filter.getDocIdSet(reader);
     if (dis == null) return null;
     disi = dis.iterator();
   } else {
     assert query != null && innerWeight != null;
     disi = innerWeight.scorer(reader, scoreDocsInOrder, topScorer);
   }
   if (disi == null) return null;
   return new ConstantScorer(similarity, disi, this);
 }
コード例 #18
0
 public ConstantScorer(Similarity similarity, IndexReader reader, Weight w) throws IOException {
   super(similarity);
   this.reader = reader;
   theScore = w.getValue();
   DocIdSet docIdSet = filter.getDocIdSet(reader);
   if (docIdSet == null) {
     _innerIter = DocIdSet.EMPTY_DOCIDSET.iterator();
   } else {
     DocIdSetIterator iter = docIdSet.iterator();
     if (iter == null) {
       _innerIter = DocIdSet.EMPTY_DOCIDSET.iterator();
     } else {
       _innerIter = iter;
     }
   }
 }
コード例 #19
0
 @Override
 public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer)
     throws IOException {
   Scorer scorer = w.scorer(reader, scoreDocsInOrder, topScorer);
   if (scorer != null) {
     // check that scorer obeys disi contract for docID() before next()/advance
     try {
       int docid = scorer.docID();
       assert docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS;
     } catch (UnsupportedOperationException ignored) {
       // from a top-level BS1
       assert topScorer;
     }
   }
   return scorer;
 }
コード例 #20
0
    @Override
    public Explanation explain(IndexReader reader, int doc) throws IOException {
      SolrIndexReader topReader = (SolrIndexReader) reader;
      SolrIndexReader[] subReaders = topReader.getLeafReaders();
      int[] offsets = topReader.getLeafOffsets();
      int readerPos = SolrIndexReader.readerIndex(doc, offsets);
      int readerBase = offsets[readerPos];

      Explanation subQueryExpl = qWeight.explain(reader, doc);
      if (!subQueryExpl.isMatch()) {
        return subQueryExpl;
      }

      DocValues vals = boostVal.getValues(context, subReaders[readerPos]);
      float sc = subQueryExpl.getValue() * vals.floatVal(doc - readerBase);
      Explanation res =
          new ComplexExplanation(true, sc, BoostedQuery.this.toString() + ", product of:");
      res.addDetail(subQueryExpl);
      res.addDetail(vals.explain(doc - readerBase));
      return res;
    }
コード例 #21
0
 @Override
 public boolean scoresDocsOutOfOrder() {
   return (innerWeight != null) ? innerWeight.scoresDocsOutOfOrder() : false;
 }
コード例 #22
0
 public ConstantScorer(Similarity similarity, IndexReader reader, Weight w) throws IOException {
   super(similarity);
   theScore = w.getValue();
   docIdSetIterator = filter.getDocIdSet(reader).iterator();
 }
コード例 #23
0
ファイル: PRMSAndQueryTest.java プロジェクト: renekrie/querqy
  @Test
  public void testGetThatFieldProbabilityRatioIsReflectedInBoost() throws Exception {

    ArgumentCaptor<Float> normalizeCaptor = ArgumentCaptor.forClass(Float.class);

    DocumentFrequencyCorrection dfc = new DocumentFrequencyCorrection();

    Directory directory = newDirectory();

    Analyzer analyzer =
        new Analyzer() {
          protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer source = new WhitespaceTokenizer();
            TokenStream filter =
                new WordDelimiterFilter(
                    source,
                    WordDelimiterFilter.GENERATE_WORD_PARTS
                        | WordDelimiterFilter.SPLIT_ON_CASE_CHANGE,
                    null);
            filter = new LowerCaseFilter(filter);
            return new TokenStreamComponents(source, filter);
          }
        };

    IndexWriterConfig conf = new IndexWriterConfig(analyzer);
    conf.setCodec(Codec.forName(TestUtil.LUCENE_CODEC));
    IndexWriter indexWriter = new IndexWriter(directory, conf);

    // Both fields f1 and f2 have 10 terms in total.
    // f1: the search terms (abc def) make 100% of all terms in f1
    // f2: the search terms (abc def) make 50% of all terms in f2
    // --> we expect that the sum of the boost factors for terms in bq(+f1:abc, +f1:def)
    // equals 2 * sum of the boost factors for terms in bq(+f2:abc, +f2:def)

    PRMSFieldBoostTest.addNumDocs("f1", "abc def", indexWriter, 2);
    PRMSFieldBoostTest.addNumDocs("f1", "abc", indexWriter, 4);
    PRMSFieldBoostTest.addNumDocs("f1", "def", indexWriter, 2);
    PRMSFieldBoostTest.addNumDocs("f2", "abc def", indexWriter, 1);
    PRMSFieldBoostTest.addNumDocs("f2", "abc", indexWriter, 2);
    PRMSFieldBoostTest.addNumDocs("f2", "def", indexWriter, 1);
    PRMSFieldBoostTest.addNumDocs("f2", "ghi", indexWriter, 5);

    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = new IndexSearcher(indexReader);
    indexSearcher.setSimilarity(similarity);

    Map<String, Float> fields = new HashMap<>();
    fields.put("f1", 1f);
    fields.put("f2", 1f);
    SearchFieldsAndBoosting searchFieldsAndBoosting =
        new SearchFieldsAndBoosting(FieldBoostModel.PRMS, fields, fields, 0.8f);

    LuceneQueryBuilder queryBuilder =
        new LuceneQueryBuilder(dfc, analyzer, searchFieldsAndBoosting, 0.01f, null);

    WhiteSpaceQuerqyParser parser = new WhiteSpaceQuerqyParser();

    Query query = queryBuilder.createQuery(parser.parse("AbcDef"));
    dfc.finishedUserQuery();

    assertTrue(query instanceof DisjunctionMaxQuery);

    DisjunctionMaxQuery dmq = (DisjunctionMaxQuery) query;
    List<Query> disjuncts = dmq.getDisjuncts();
    assertEquals(2, disjuncts.size());

    Query disjunct1 = disjuncts.get(0);
    if (disjunct1 instanceof BoostQuery) {
      disjunct1 = ((BoostQuery) disjunct1).getQuery();
    }
    assertTrue(disjunct1 instanceof BooleanQuery);

    BooleanQuery bq1 = (BooleanQuery) disjunct1;

    Query disjunct2 = disjuncts.get(1);
    if (disjunct2 instanceof BoostQuery) {
      disjunct2 = ((BoostQuery) disjunct2).getQuery();
    }
    assertTrue(disjunct2 instanceof BooleanQuery);

    BooleanQuery bq2 = (BooleanQuery) disjunct2;

    final Weight weight1 = bq1.createWeight(indexSearcher, true);
    weight1.normalize(0.1f, 4f);

    final Weight weight2 = bq2.createWeight(indexSearcher, true);
    weight2.normalize(0.1f, 4f);

    Mockito.verify(simWeight, times(4)).normalize(eq(0.1f), normalizeCaptor.capture());
    final List<Float> capturedBoosts = normalizeCaptor.getAllValues();

    // capturedBoosts = boosts of [bq1.term1, bq1.term2, bq2.term1, bq2.term2 ]
    assertEquals(capturedBoosts.get(0), capturedBoosts.get(1), 0.00001);
    assertEquals(capturedBoosts.get(2), capturedBoosts.get(3), 0.00001);
    assertEquals(2f, capturedBoosts.get(0) / capturedBoosts.get(3), 0.00001);

    indexReader.close();
    directory.close();
    analyzer.close();
  }
コード例 #24
0
 @Override
 public void normalize(float norm) {
   norm *= getBoost();
   qWeight.normalize(norm);
 }
コード例 #25
0
 @Override
 public float sumOfSquaredWeights() throws IOException {
   float sum = qWeight.sumOfSquaredWeights();
   sum *= getBoost() * getBoost();
   return sum;
 }
コード例 #26
0
 /** Apply the computed normalization factor to our subqueries */
 @Override
 public void normalize(float norm, float boost) {
   for (Weight wt : weights) {
     wt.normalize(norm, boost);
   }
 }
コード例 #27
0
ファイル: DTrainTest.java プロジェクト: aglne/shifu
  @Test
  public void quickTest() throws IOException {

    double[] gradientError = new double[NUM_EPOCHS];
    double[] ecogError = new double[NUM_EPOCHS];

    network.reset();
    weights = network.getFlat().getWeights();

    MLDataSet[] subsets = splitDataSet(training);
    Gradient[] workers = new Gradient[numSplit];

    Weight weightCalculator = null;

    for (int i = 0; i < workers.length; i++) {
      workers[i] = initGradient(subsets[i]);
      workers[i].setWeights(weights);
    }

    log.info("Running QuickPropagtaion testing! ");
    NNParams globalParams = new NNParams();
    globalParams.setWeights(weights);

    for (int i = 0; i < NUM_EPOCHS; i++) {

      double error = 0.0;

      // each worker do the job
      for (int j = 0; j < workers.length; j++) {
        workers[j].run();
        error += workers[j].getError();
      }

      gradientError[i] = error / workers.length;

      log.info("The #" + i + " training error: " + gradientError[i]);

      // master
      globalParams.reset();

      for (int j = 0; j < workers.length; j++) {
        globalParams.accumulateGradients(workers[j].getGradients());
        globalParams.accumulateTrainSize(subsets[j].getRecordCount());
      }

      if (weightCalculator == null) {
        weightCalculator =
            new Weight(
                globalParams.getGradients().length,
                globalParams.getTrainSize(),
                this.rate,
                NNUtils.QUICK_PROPAGATION);
      }

      double[] interWeight =
          weightCalculator.calculateWeights(globalParams.getWeights(), globalParams.getGradients());

      globalParams.setWeights(interWeight);

      // set weights
      for (int j = 0; j < workers.length; j++) {
        workers[j].setWeights(interWeight);
      }
    }

    // encog
    network.reset();
    // NNUtils.randomize(numSplit, weights);
    network.getFlat().setWeights(weights);

    Propagation p = null;
    p = new QuickPropagation(network, training, rate);
    // p = new ManhattanPropagation(network, training, rate);
    p.setThreadCount(numSplit);

    for (int i = 0; i < NUM_EPOCHS; i++) {
      p.iteration(1);
      // System.out.println("the #" + i + " training error: " + p.getError());
      ecogError[i] = p.getError();
    }

    // assert
    double diff = 0.0;
    for (int i = 0; i < NUM_EPOCHS; i++) {
      diff += Math.abs(ecogError[i] - gradientError[i]);
    }

    Assert.assertTrue(diff / NUM_EPOCHS < 0.1);
  }
コード例 #28
0
 @Override
 public void extractTerms(Set<Term> terms) {
   for (Weight weight : weights) {
     weight.extractTerms(terms);
   }
 }
コード例 #29
0
 public ConstantScorer(Similarity similarity, DocIdSetIterator docIdSetIterator, Weight w)
     throws IOException {
   super(similarity, w);
   theScore = w.getValue();
   this.docIdSetIterator = docIdSetIterator;
 }
コード例 #30
0
 @Override
 public float getValueForNormalization() throws IOException {
   float sum = parentWeight.getValueForNormalization();
   sum *= getBoost() * getBoost();
   return sum;
 }