コード例 #1
0
  // LUCENE-325: test forceMergeDeletes without waiting, when
  // many adjacent merges are required
  public void testForceMergeDeletes3() throws IOException {
    Directory dir = newDirectory();
    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMaxBufferedDocs(2)
                .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
                .setMergePolicy(newLogMergePolicy(50)));

    FieldType customType = new FieldType();
    customType.setStored(true);

    FieldType customType1 = new FieldType(TextField.TYPE_NOT_STORED);
    customType1.setTokenized(false);
    customType1.setStoreTermVectors(true);
    customType1.setStoreTermVectorPositions(true);
    customType1.setStoreTermVectorOffsets(true);

    Document document = new Document();
    Field storedField = newField("stored", "stored", customType);
    document.add(storedField);
    Field termVectorField = newField("termVector", "termVector", customType1);
    document.add(termVectorField);
    Field idField = newStringField("id", "", Field.Store.NO);
    document.add(idField);
    for (int i = 0; i < 98; i++) {
      idField.setStringValue("" + i);
      writer.addDocument(document);
    }
    writer.close();

    IndexReader ir = DirectoryReader.open(dir);
    assertEquals(98, ir.maxDoc());
    assertEquals(98, ir.numDocs());
    ir.close();

    IndexWriterConfig dontMergeConfig =
        new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
    writer = new IndexWriter(dir, dontMergeConfig);
    for (int i = 0; i < 98; i += 2) {
      writer.deleteDocuments(new Term("id", "" + i));
    }
    writer.close();
    ir = DirectoryReader.open(dir);
    assertEquals(49, ir.numDocs());
    ir.close();

    writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy(3)));
    writer.forceMergeDeletes(false);
    writer.close();
    ir = DirectoryReader.open(dir);
    assertEquals(49, ir.maxDoc());
    assertEquals(49, ir.numDocs());
    ir.close();
    dir.close();
  }
コード例 #2
0
  public static SimpleOrderedMap<Object> getIndexInfo(IndexReader reader, boolean countTerms)
      throws IOException {
    Directory dir = reader.directory();
    SimpleOrderedMap<Object> indexInfo = new SimpleOrderedMap<Object>();

    indexInfo.add("numDocs", reader.numDocs());
    indexInfo.add("maxDoc", reader.maxDoc());

    if (countTerms) {
      TermEnum te = null;
      try {
        te = reader.terms();
        int numTerms = 0;
        while (te.next()) {
          numTerms++;
        }
        indexInfo.add("numTerms", numTerms);
      } finally {
        if (te != null) te.close();
      }
    }

    indexInfo.add(
        "version",
        reader.getVersion()); // TODO? Is this different then: IndexReader.getCurrentVersion( dir )?
    indexInfo.add("optimized", reader.isOptimized());
    indexInfo.add("current", reader.isCurrent());
    indexInfo.add("hasDeletions", reader.hasDeletions());
    indexInfo.add("directory", dir);
    indexInfo.add("lastModified", new Date(IndexReader.lastModified(dir)));
    return indexInfo;
  }
コード例 #3
0
 public Query percolateQuery(
     String documentType,
     PercolateQuery.QueryStore queryStore,
     BytesReference documentSource,
     IndexSearcher searcher)
     throws IOException {
   IndexReader indexReader = searcher.getIndexReader();
   Query candidateMatchesQuery = createCandidateQuery(indexReader);
   Query verifiedMatchesQuery;
   // We can only skip the MemoryIndex verification when percolating a single document.
   // When the document being percolated contains a nested object field then the MemoryIndex
   // contains multiple
   // documents. In this case the term query that indicates whether memory index verification can
   // be skipped
   // can incorrectly indicate that non nested queries would match, while their nested variants
   // would not.
   if (indexReader.maxDoc() == 1) {
     verifiedMatchesQuery =
         new TermQuery(new Term(extractionResultField.name(), EXTRACTION_COMPLETE));
   } else {
     verifiedMatchesQuery = new MatchNoDocsQuery("nested docs, so no verified matches");
   }
   return new PercolateQuery(
       documentType,
       queryStore,
       documentSource,
       candidateMatchesQuery,
       searcher,
       verifiedMatchesQuery);
 }
コード例 #4
0
  @Override
  public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
    StockAction action = getStockAction(); // retrieve the service
    long lastUpdateTime = action.geLastUpdateTime();
    if (lastUpdateTime != this.lastUpdateTime) {
      cache.clear(); // clear outdated cache
    }
    DocIdSet cached = cache.get(reader); // check if in cache already
    if (cached != null) return cached;
    // not in cache, build info
    final BitSet bitSet = getAllPositiveBitSet(reader.maxDoc()); // by default, all documents pass

    Term clazzTerm = new Term(DocumentBuilder.CLASS_FIELDNAME, Item.class.getName());
    if (reader.docFreq(clazzTerm) == 0) { // no need to filter
      // index does not contain Item objects
      // no-op
    } else {
      // for each item out of stock, find the corresponding document id by item id
      // and switch off the corresponding bit
      for (String ean : action.getEanOfItemsOutOfStock()) { // invoke external service
        Term term = new Term("ean", ean);
        TermDocs termDocs = reader.termDocs(term); // find document by ean
        while (termDocs.next()) {
          bitSet.clear(termDocs.doc());
        }
      }
    }
    DocIdSet docIdSet = new DocIdBitSet(bitSet); // build DocIdSet from BitSet
    cache.put(reader, docIdSet); // put results in the cache
    this.lastUpdateTime = lastUpdateTime; // update timestamp
    return docIdSet;
  }
コード例 #5
0
 @Override
 public void collect(int doc) throws IOException {
   BytesWrap parentId = typeCache.parentIdByDoc(doc);
   if (parentId == null) {
     return;
   }
   for (Tuple<IndexReader, IdReaderTypeCache> tuple : readers) {
     IndexReader indexReader = tuple.v1();
     IdReaderTypeCache idReaderTypeCache = tuple.v2();
     if (idReaderTypeCache
         == null) { // might be if we don't have that doc with that type in this reader
       continue;
     }
     int parentDocId = idReaderTypeCache.docById(parentId);
     if (parentDocId != -1 && !indexReader.isDeleted(parentDocId)) {
       OpenBitSet docIdSet = parentDocs().get(indexReader.getCoreCacheKey());
       if (docIdSet == null) {
         docIdSet = new OpenBitSet(indexReader.maxDoc());
         parentDocs.put(indexReader.getCoreCacheKey(), docIdSet);
       }
       docIdSet.fastSet(parentDocId);
       return;
     }
   }
 }
コード例 #6
0
 @Override
 public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
   DocSet boundingBoxDocSet = null;
   if (boundingBoxFilter != null) {
     DocIdSet docIdSet = boundingBoxFilter.getDocIdSet(reader);
     if (docIdSet == null) {
       return null;
     }
     boundingBoxDocSet = DocSets.convert(reader, docIdSet);
   }
   final GeoPointFieldData fieldData =
       (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, reader, fieldName);
   GeoDistanceRangeDocSet distDocSet =
       new GeoDistanceRangeDocSet(
           reader.maxDoc(),
           fieldData,
           fixedSourceDistance,
           distanceBoundingCheck,
           inclusiveLowerPoint,
           inclusiveUpperPoint);
   if (boundingBoxDocSet == null) {
     return distDocSet;
   } else {
     return new AndDocSet(ImmutableList.of(boundingBoxDocSet, distDocSet));
   }
 }
コード例 #7
0
  public ArrayList<String> filterSearchQueryString(ArrayList<String> qstring) throws IOException {
    Term vword = new Term("");
    int docFreq = indexReader.docFreq(vword);
    int numDocs = indexReader.maxDoc();
    float idf = (float) Math.log(numDocs / (double) (docFreq + 1) + 1.0);

    return qstring;
  }
コード例 #8
0
 protected ValueSourceScorer(IndexReader reader, DocValues values) {
   super(null);
   this.reader = reader;
   this.maxDoc = reader.maxDoc();
   this.values = values;
   setCheckDeletes(true);
   this.delDocs = MultiFields.getDeletedDocs(reader);
 }
コード例 #9
0
 public AllScorer(Similarity similarity, IndexReader reader, FunctionWeight w)
     throws IOException {
   super(similarity, w);
   this.weight = w;
   this.qWeight = w.getValue();
   this.reader = reader;
   this.maxDoc = reader.maxDoc();
   this.hasDeletions = reader.hasDeletions();
   vals = func.getValues(weight.context, reader);
 }
コード例 #10
0
ファイル: TestJoinUtil.java プロジェクト: pengzong1111/solr4
  private FixedBitSet createExpectedResult(
      String queryValue, boolean from, IndexReader topLevelReader, IndexIterationContext context)
      throws IOException {
    final Map<String, List<RandomDoc>> randomValueDocs;
    final Map<String, List<RandomDoc>> linkValueDocuments;
    if (from) {
      randomValueDocs = context.randomValueFromDocs;
      linkValueDocuments = context.toDocuments;
    } else {
      randomValueDocs = context.randomValueToDocs;
      linkValueDocuments = context.fromDocuments;
    }

    FixedBitSet expectedResult = new FixedBitSet(topLevelReader.maxDoc());
    List<RandomDoc> matchingDocs = randomValueDocs.get(queryValue);
    if (matchingDocs == null) {
      return new FixedBitSet(topLevelReader.maxDoc());
    }

    for (RandomDoc matchingDoc : matchingDocs) {
      for (String linkValue : matchingDoc.linkValues) {
        List<RandomDoc> otherMatchingDocs = linkValueDocuments.get(linkValue);
        if (otherMatchingDocs == null) {
          continue;
        }

        for (RandomDoc otherSideDoc : otherMatchingDocs) {
          DocsEnum docsEnum =
              MultiFields.getTermDocsEnum(
                  topLevelReader,
                  MultiFields.getLiveDocs(topLevelReader),
                  "id",
                  new BytesRef(otherSideDoc.id),
                  0);
          assert docsEnum != null;
          int doc = docsEnum.nextDoc();
          expectedResult.set(doc);
        }
      }
    }
    return expectedResult;
  }
コード例 #11
0
  public void testScoringNoBaseQuery() throws IOException {
    // verify that drill-down queries (with no base query) returns 0.0 score
    IndexSearcher searcher = newSearcher(reader);

    DrillDownQuery q = new DrillDownQuery(config);
    q.add("a");
    TopDocs docs = searcher.search(q, reader.maxDoc()); // fetch all available docs to this query
    for (ScoreDoc sd : docs.scoreDocs) {
      assertEquals(0f, sd.score, 0f);
    }
  }
コード例 #12
0
  public void testScoring() throws IOException {
    // verify that drill-down queries do not modify scores
    IndexSearcher searcher = newSearcher(reader);

    float[] scores = new float[reader.maxDoc()];

    Query q = new TermQuery(new Term("content", "foo"));
    TopDocs docs = searcher.search(q, reader.maxDoc()); // fetch all available docs to this query
    for (ScoreDoc sd : docs.scoreDocs) {
      scores[sd.doc] = sd.score;
    }

    // create a drill-down query with category "a", scores should not change
    DrillDownQuery q2 = new DrillDownQuery(config, q);
    q2.add("a");
    docs = searcher.search(q2, reader.maxDoc()); // fetch all available docs to this query
    for (ScoreDoc sd : docs.scoreDocs) {
      assertEquals("score of doc=" + sd.doc + " modified", scores[sd.doc], sd.score, 0f);
    }
  }
コード例 #13
0
  @Override
  public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
    LOG.info("build()");
    if (sourceLocation == null) {
      reader = searcher.getIndexReader();
      dictionary = new HighFrequencyDictionary(reader, field, threshold);
    } else {
      try {

        final String fileDelim = ",";
        if (sourceLocation.contains(fileDelim)) {
          String[] files = sourceLocation.split(fileDelim);
          Reader[] readers = new Reader[files.length];
          for (int i = 0; i < files.length; i++) {
            Reader reader =
                new InputStreamReader(
                    core.getResourceLoader().openResource(files[i]), IOUtils.CHARSET_UTF_8);
            readers[i] = reader;
          }
          dictionary = new MultipleFileDictionary(readers);
        } else {
          dictionary =
              new FileDictionary(
                  new InputStreamReader(
                      core.getResourceLoader().openResource(sourceLocation),
                      IOUtils.CHARSET_UTF_8));
        }
      } catch (UnsupportedEncodingException e) {
        // should not happen
        LOG.error("should not happen", e);
      }
    }

    lookup.build(dictionary);
    if (storeDir != null) {
      File target = new File(storeDir, factory.storeFileName());
      if (!lookup.store(new FileOutputStream(target))) {
        if (sourceLocation == null) {
          assert reader != null && field != null;
          LOG.error(
              "Store Lookup build from index on field: "
                  + field
                  + " failed reader has: "
                  + reader.maxDoc()
                  + " docs");
        } else {
          LOG.error("Store Lookup build from sourceloaction: " + sourceLocation + " failed");
        }
      } else {
        LOG.info("Stored suggest data to: " + target.getAbsolutePath());
      }
    }
  }
コード例 #14
0
    // public SortingReader(IndexReader oldReader, int[] oldToNew) { // TODO MC
    public SortingReader(IndexReader oldReader, DocScore[] newToOld) { // TODO MC
      super(oldReader);
      this.newToOld = newToOld;

      this.oldToNew = new int[oldReader.maxDoc()];
      int newDoc = 0;
      while (newDoc < newToOld.length) {
        int oldDoc = newToOld[newDoc].doc;
        oldToNew[oldDoc] = newDoc;
        newDoc++;
      }
    }
コード例 #15
0
  @Test
  public void testSimple() throws Exception {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);

    for (int parent = 1; parent <= 5; parent++) {
      Document document = new Document();
      document.add(
          new StringField(
              UidFieldMapper.NAME,
              Uid.createUid("parent", Integer.toString(parent)),
              Field.Store.NO));
      document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO));
      indexWriter.addDocument(document);

      for (int child = 1; child <= 3; child++) {
        document = new Document();
        document.add(
            new StringField(
                UidFieldMapper.NAME,
                Uid.createUid("child", Integer.toString(parent * 3 + child)),
                Field.Store.NO));
        document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO));
        document.add(
            new StringField(
                ParentFieldMapper.NAME,
                Uid.createUid("parent", Integer.toString(parent)),
                Field.Store.NO));
        document.add(new StringField("field1", "value" + child, Field.Store.NO));
        indexWriter.addDocument(document);
      }
    }

    IndexReader indexReader = DirectoryReader.open(indexWriter.w, false);
    IndexSearcher searcher = new IndexSearcher(indexReader);

    TermQuery childQuery = new TermQuery(new Term("field1", "value" + (1 + random().nextInt(3))));
    TermFilter parentFilter = new TermFilter(new Term(TypeFieldMapper.NAME, "parent"));
    int shortCircuitParentDocSet = random().nextInt(5);
    ChildrenConstantScoreQuery query =
        new ChildrenConstantScoreQuery(
            childQuery, "parent", "child", parentFilter, shortCircuitParentDocSet, null);

    BitSetCollector collector = new BitSetCollector(indexReader.maxDoc());
    searcher.search(query, collector);
    FixedBitSet actualResult = collector.getResult();

    assertThat(actualResult.cardinality(), equalTo(5));

    indexWriter.close();
    indexReader.close();
    directory.close();
  }
コード例 #16
0
 @Override
 public Query rewrite(IndexReader reader) throws IOException {
   Term term = new Term(CONTENT_TYPE, type);
   TermContext context = TermContext.build(reader.getContext(), term);
   if (context.docFreq() == reader.maxDoc()) {
     // All docs have the same type.
     // Using a match_all query will help Lucene perform some optimizations
     // For instance, match_all queries as filter clauses are automatically removed
     return new MatchAllDocsQuery();
   } else {
     return new ConstantScoreQuery(new TermQuery(term, context));
   }
 }
コード例 #17
0
  public void getIndexInfo(String indexdir, int freqThreshold) {
    IndexReader reader = null;

    try {
      Directory dir = FSDirectory.open(new File(indexdir));
      System.out.println(dir);
      reader = IndexReader.open(dir);

      System.out.println("document num:" + reader.numDocs());
      System.out.println("======================");

      TermEnum terms = reader.terms();
      sortedTermQueue.clear();
      maxDocNum = reader.maxDoc();
      linkMap.clear();
      termList.clear();
      while (terms.next()) {
        // System.out.print(terms.term() + "\tDocFreq:" +
        TermDocs termDocs = reader.termDocs(terms.term());
        MyTerm temp = new MyTerm(terms.term(), termDocs, maxDocNum);
        if (temp.totalFreq < freqThreshold) {
          continue;
        } /*
           * if(temp.originTrem.text().length()==1){ continue; }
           */
        linkMap.put(temp.originTrem.text(), temp);
        sortedTermQueue.add(temp);
        termList.add(temp);
      }
      System.out.println("total Size:" + sortedTermQueue.size());
      System.out.println("mapsize:" + linkMap.keySet().size());
      // System.exit(0);
      int num = 0;
      this.maxFreq = sortedTermQueue.peek().totalFreq;
      while (!sortedTermQueue.isEmpty()) {
        num++;
        System.out.println(num + ":" + sortedTermQueue.poll());
      }
      System.out.println("read index info done");
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      try {
        reader.close();

      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }
コード例 #18
0
ファイル: IndexUtil.java プロジェクト: joy-xu/lucene_test
 public void query() {
   try {
     IndexReader reader = IndexReader.open(directory);
     // 通过reader可以有效的获取到文档的数量
     System.out.println("numDocs:" + reader.numDocs());
     System.out.println("maxDocs:" + reader.maxDoc());
     System.out.println("deleteDocs:" + reader.numDeletedDocs());
     reader.close();
   } catch (CorruptIndexException e) {
     e.printStackTrace();
   } catch (IOException e) {
     e.printStackTrace();
   }
 }
コード例 #19
0
  public void testRandom() throws IOException {
    Directory dir = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    int num = atLeast(100);
    List<Term> terms = new ArrayList<>();
    for (int i = 0; i < num; i++) {
      String field = "field" + i;
      String string = TestUtil.randomRealisticUnicodeString(random());
      terms.add(new Term(field, string));
      Document doc = new Document();
      doc.add(newStringField(field, string, Field.Store.NO));
      w.addDocument(doc);
    }
    IndexReader reader = w.getReader();
    w.close();

    IndexSearcher searcher = newSearcher(reader);

    int numQueries = atLeast(10);
    for (int i = 0; i < numQueries; i++) {
      Term term = terms.get(random().nextInt(num));
      TopDocs queryResult = searcher.search(new TermQuery(term), reader.maxDoc());

      MatchAllDocsQuery matchAll = new MatchAllDocsQuery();
      final TermFilter filter = termFilter(term);
      TopDocs filterResult = searcher.search(matchAll, filter, reader.maxDoc());
      assertEquals(filterResult.totalHits, queryResult.totalHits);
      ScoreDoc[] scoreDocs = filterResult.scoreDocs;
      for (int j = 0; j < scoreDocs.length; j++) {
        assertEquals(scoreDocs[j].doc, queryResult.scoreDocs[j].doc);
      }
    }

    reader.close();
    dir.close();
  }
コード例 #20
0
    private int run(int base, IndexReader reader) throws IOException {
      IndexReader[] subReaders = reader.getSequentialSubReaders();
      if (subReaders == null) {
        // atomic reader
        add(base, reader);
        base += reader.maxDoc();
      } else {
        // composite reader
        for (int i = 0; i < subReaders.length; i++) {
          base = run(base, subReaders[i]);
        }
      }

      return base;
    }
コード例 #21
0
ファイル: AdaptiveSearcher.java プロジェクト: naryad/Solr4.0
  /**
   * Search with facets through the {@link AdaptiveFacetsAccumulator}
   *
   * @param indexDir Directory of the search index.
   * @param taxoDir Directory of the taxonomy index.
   * @throws Exception on error (no detailed exception handling here for sample simplicity
   * @return facet results
   */
  public static List<FacetResult> searchWithFacets(Directory indexDir, Directory taxoDir)
      throws Exception {
    // prepare index reader and taxonomy.
    TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);
    IndexReader indexReader = DirectoryReader.open(indexDir);

    // prepare searcher to search against
    IndexSearcher searcher = new IndexSearcher(indexReader);

    // faceted search is working in 2 steps:
    // 1. collect matching documents
    // 2. aggregate facets for collected documents and
    //    generate the requested faceted results from the aggregated facets

    // step 1: collect matching documents into a collector
    Query q = new TermQuery(new Term(SimpleUtils.TEXT, "white"));
    ExampleUtils.log("Query: " + q);

    // regular collector for scoring matched documents
    TopScoreDocCollector topDocsCollector = TopScoreDocCollector.create(10, true);

    // docids collector for guiding facets accumulation (scoring disabled)
    ScoredDocIdCollector docIdsCollecor = ScoredDocIdCollector.create(indexReader.maxDoc(), false);

    // Faceted search parameters indicate which facets are we interested in
    FacetSearchParams facetSearchParams = new FacetSearchParams();
    facetSearchParams.addFacetRequest(new CountFacetRequest(new CategoryPath("root", "a"), 10));

    // search, into both collectors. note: in case only facets accumulation
    // is required, the topDocCollector part can be totally discarded
    searcher.search(q, MultiCollector.wrap(topDocsCollector, docIdsCollecor));

    // Obtain facets results and print them
    AdaptiveFacetsAccumulator accumulator =
        new AdaptiveFacetsAccumulator(facetSearchParams, indexReader, taxo);
    List<FacetResult> res = accumulator.accumulate(docIdsCollecor.getScoredDocIDs());

    int i = 0;
    for (FacetResult facetResult : res) {
      ExampleUtils.log("Res " + (i++) + ": " + facetResult);
    }

    // we're done, close the index reader and the taxonomy.
    indexReader.close();
    taxo.close();

    return res;
  }
コード例 #22
0
  public SubReaderFilter(IndexReader reader, DocIdSet docs) {
    this.docs = docs;

    IndexReader[] subreaders = reader.getSequentialSubReaders();
    if (subreaders != null && subreaders.length > 0) {
      bases = Maps.newHashMap();

      // Get the starting point of each subreader
      int docBase = 0;
      for (IndexReader sub : subreaders) {
        int readerSize = sub.maxDoc();
        bases.put(sub, docBase);
        docBase += readerSize;
      }
    }
  }
コード例 #23
0
 /*
  * LUCENE-6006: Test undead norms.
  *                                 .....
  *                             C C  /
  *                            /<   /
  *             ___ __________/_#__=o
  *            /(- /(\_\________   \
  *            \ ) \ )_      \o     \
  *            /|\ /|\       |'     |
  *                          |     _|
  *                          /o   __\
  *                         / '     |
  *                        / /      |
  *                       /_/\______|
  *                      (   _(    <
  *                       \    \    \
  *                        \    \    |
  *                         \____\____\
  *                         ____\_\__\_\
  *                       /`   /`     o\
  *                       |___ |_______|
  *
  */
 public void testReadUndeadNorms() throws Exception {
   InputStream resource =
       TestLucene42NormsFormat.class.getResourceAsStream("index.42.undeadnorms.zip");
   assertNotNull(resource);
   Path path = createTempDir("undeadnorms");
   TestUtil.unzip(resource, path);
   Directory dir = FSDirectory.open(path);
   IndexReader r = DirectoryReader.open(dir);
   NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
   assertNotNull(undeadNorms);
   assertEquals(2, r.maxDoc());
   assertEquals(0, undeadNorms.get(0));
   assertEquals(0, undeadNorms.get(1));
   dir.close();
   r.close();
 }
コード例 #24
0
  /**
   * Indexes at least 1 document with f1, and at least 1 document with f2. returns the norms for
   * "field".
   */
  NumericDocValues getNorms(String field, Field f1, Field f2) throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig iwc =
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
            .setMergePolicy(newLogMergePolicy());
    RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);

    // add f1
    Document d = new Document();
    d.add(f1);
    riw.addDocument(d);

    // add f2
    d = new Document();
    d.add(f2);
    riw.addDocument(d);

    // add a mix of f1's and f2's
    int numExtraDocs = TestUtil.nextInt(random(), 1, 1000);
    for (int i = 0; i < numExtraDocs; i++) {
      d = new Document();
      d.add(random().nextBoolean() ? f1 : f2);
      riw.addDocument(d);
    }

    IndexReader ir1 = riw.getReader();
    // todo: generalize
    NumericDocValues norms1 = MultiDocValues.getNormValues(ir1, field);

    // fully merge and validate MultiNorms against single segment.
    riw.forceMerge(1);
    DirectoryReader ir2 = riw.getReader();
    NumericDocValues norms2 = getOnlySegmentReader(ir2).getNormValues(field);

    if (norms1 == null) {
      assertNull(norms2);
    } else {
      for (int docID = 0; docID < ir1.maxDoc(); docID++) {
        assertEquals(norms1.get(docID), norms2.get(docID));
      }
    }
    ir1.close();
    ir2.close();
    riw.close();
    dir.close();
    return norms1;
  }
コード例 #25
0
 @Override
 public Query rewrite(IndexReader reader) throws IOException {
   if (getBoost() != 1f) {
     return super.rewrite(reader);
   }
   if (this.terms.isEmpty()) {
     return new MatchNoDocsQuery();
   } else if (this.terms.size() == 1) {
     return newTermQuery(this.terms.get(0), null);
   }
   final List<LeafReaderContext> leaves = reader.leaves();
   final int maxDoc = reader.maxDoc();
   final TermContext[] contextArray = new TermContext[terms.size()];
   final Term[] queryTerms = this.terms.toArray(new Term[0]);
   collectTermContext(reader, leaves, contextArray, queryTerms);
   return buildQuery(maxDoc, contextArray, queryTerms);
 }
コード例 #26
0
ファイル: TestDocCount.java プロジェクト: PATRIC3/p3_solr
 private void verifyCount(IndexReader ir) throws Exception {
   Fields fields = MultiFields.getFields(ir);
   for (String field : fields) {
     Terms terms = fields.terms(field);
     if (terms == null) {
       continue;
     }
     int docCount = terms.getDocCount();
     FixedBitSet visited = new FixedBitSet(ir.maxDoc());
     TermsEnum te = terms.iterator();
     while (te.next() != null) {
       PostingsEnum de = TestUtil.docs(random(), te, null, PostingsEnum.NONE);
       while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
         visited.set(de.docID());
       }
     }
     assertEquals(visited.cardinality(), docCount);
   }
 }
コード例 #27
0
 @Override
 public FieldStats.Long stats(IndexReader reader) throws IOException {
   int maxDoc = reader.maxDoc();
   Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
   if (terms == null) {
     return null;
   }
   long minValue = LegacyNumericUtils.getMinInt(terms);
   long maxValue = LegacyNumericUtils.getMaxInt(terms);
   return new FieldStats.Long(
       maxDoc,
       terms.getDocCount(),
       terms.getSumDocFreq(),
       terms.getSumTotalTermFreq(),
       isSearchable(),
       isAggregatable(),
       minValue,
       maxValue);
 }
コード例 #28
0
 @Override
 public FieldStats.Ip stats(IndexReader reader) throws IOException {
   String field = name();
   long size = XPointValues.size(reader, field);
   if (size == 0) {
     return null;
   }
   int docCount = XPointValues.getDocCount(reader, field);
   byte[] min = XPointValues.getMinPackedValue(reader, field);
   byte[] max = XPointValues.getMaxPackedValue(reader, field);
   return new FieldStats.Ip(
       reader.maxDoc(),
       docCount,
       -1L,
       size,
       isSearchable(),
       isAggregatable(),
       InetAddressPoint.decode(min),
       InetAddressPoint.decode(max));
 }
 /*     */ public DocIdSet getDocIdSet(IndexReader reader) /*     */ throws IOException /*     */ {
   /* 103 */ TermEnum enumerator = this.query.getEnum(reader);
   /*     */ try
   /*     */ {
     /* 106 */ if (enumerator.term() == null) {
       /* 107 */ return DocIdSet.EMPTY_DOCIDSET;
     }
     OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
     /* 110 */ int[] docs = new int[32];
     /* 111 */ int[] freqs = new int[32];
     /* 112 */ TermDocs termDocs = reader.termDocs();
     /*     */ int termCount;
     /*     */ try {
       termCount = 0;
       /*     */ do {
         /* 116 */ Term term = enumerator.term();
         /* 117 */ if (term == null) /*     */ break;
         /* 119 */ termCount++;
         /* 120 */ termDocs.seek(term);
         /*     */ while (true) {
           /* 122 */ int count = termDocs.read(docs, freqs);
           /* 123 */ if (count == 0) break;
           /* 124 */ for (int i = 0; i < count; i++) {
             /* 125 */ bitSet.set(docs[i]);
             /*     */ }
           /*     */ }
         /*     */
         /*     */ }
       /*     */
       /* 131 */ while (enumerator.next());
       /*     */
       /* 133 */ this.query.incTotalNumberOfTerms(termCount);
       /*     */ } finally
     /*     */ {
       /* 136 */ termDocs.close();
       /*     */ }
     /* 138 */ return bitSet;
     /*     */ } finally {
     /* 140 */ enumerator.close();
     /*     */ }
   /*     */ }
コード例 #30
0
  @Override
  public DocIdSet getDocIdSet(IndexReader subReader) throws IOException {

    if (bases == null || !bases.containsKey(subReader)) {
      return docs;
    }

    int docBase = bases.get(subReader);

    int readerSize = subReader.maxDoc();
    OpenBitSet filter = new OpenBitSet(readerSize);

    DocIdSetIterator iterator = docs.iterator();
    int doc = iterator.advance(docBase);

    while (doc < docBase + readerSize) {
      filter.set(doc - docBase);
      doc = iterator.nextDoc();
    }
    return filter;
  }