private void addPoint(IndexWriter writer, String name, int year, int month, int day)
     throws IOException {
   LgteDocumentWrapper doc = new LgteDocumentWrapper();
   ++i;
   doc.indexText("id", "" + i);
   doc.indexText("name", name);
   doc.addTimeField(year, month, day);
   doc.indexText("metafile", "doc");
   ((LgteIndexWriter) writer).addDocument(doc);
 }
  public void testTimeFilter() throws IOException, DocumentException, ParseException {
    LgteIndexWriter writer = new LgteIndexWriter(path + "Contents", true);
    LgteDocumentWrapper doc1 = new LgteDocumentWrapper();
    doc1.indexText(Globals.DOCUMENT_ID_FIELD, "1");
    doc1.indexText("contents", "word1 word2 word3");
    doc1.indexStringNoStore(Config.S_HAS_TIMEXES, "true");
    LgteDocumentWrapper doc2 = new LgteDocumentWrapper();
    doc2.indexText(Globals.DOCUMENT_ID_FIELD, "2");
    doc2.indexText("contents", "word2 word3 word4 word55 word96 word2 word54 word33 wordss");
    writer.addDocument(doc1);
    writer.addDocument(doc2);
    writer.close();

    LgteIndexWriter writer2 = new LgteIndexWriter(path + "Sentences", true);
    LgteDocumentWrapper sentence0 = new LgteDocumentWrapper();
    sentence0.indexText(Globals.DOCUMENT_ID_FIELD, "1_0");
    sentence0.indexText("doc_id", "1");
    sentence0.indexText("sentences", "word1 word3");
    LgteDocumentWrapper sentence1 = new LgteDocumentWrapper();
    sentence1.indexText(Globals.DOCUMENT_ID_FIELD, "1_1");
    sentence1.indexText("doc_id", "1");
    sentence1.indexText("sentences", "word1 word2 word3");
    LgteDocumentWrapper sentence2 = new LgteDocumentWrapper();
    sentence2.indexStringNoStore(Config.S_HAS_TIMEXES + "_sentences", "true");
    sentence2.indexText(Globals.DOCUMENT_ID_FIELD, "2_1");
    sentence2.indexText("doc_id", "2");
    sentence2.indexText("sentences", "word2 word3 word4 word55 word96 word2 word54 word33 wordss");
    writer2.addDocument(sentence0);
    writer2.addDocument(sentence1);
    writer2.addDocument(sentence2);
    writer2.close();

    IndexReader readerContents =
        LgteIndexManager.openReader(path + "Contents", Model.OkapiBM25Model);
    IndexReader readerSentences =
        LgteIndexManager.openReader(path + "Sentences", Model.OkapiBM25Model);
    Map<String, IndexReader> readers = new HashMap<String, IndexReader>();
    readers.put("contents", readerContents);
    readers.put("sentences", readerSentences);
    readers.put(Config.S_HAS_TIMEXES, readerContents);
    readers.put(Config.S_HAS_TIMEXES + "_sentences", readerSentences);
    readers.put("doc_id", readerSentences);
    readers.put("id", readerSentences);
    LgteIsolatedIndexReader lgteIsolatedIndexReader = new LgteIsolatedIndexReader(readers);
    lgteIsolatedIndexReader.addTreeMapping(readerContents, readerSentences, "doc_id");

    LgteIndexSearcherWrapper searcher =
        new LgteIndexSearcherWrapper(Model.OkapiBM25Model, lgteIsolatedIndexReader);
    QueryConfiguration queryConfiguration = new QueryConfiguration();
    queryConfiguration.setProperty("bm25.idf.policy", "floor_epslon");
    queryConfiguration.setProperty("bm25.idf.epslon", "0.01");
    queryConfiguration.setProperty("bm25.k1", "2.0");
    queryConfiguration.setProperty("bm25.b", "0.75");
    queryConfiguration.setProperty("index.tree", "true");
    QueryFilter queryFilter =
        new QueryFilter(
            org.apache.lucene.queryParser.QueryParser.parse(
                "true", Config.S_HAS_TIMEXES, new LgteNothingAnalyzer()));
    LgteQuery lgteQuery =
        LgteQueryParser.parseQuery(
            "sentences:word2", new LgteNothingAnalyzer(), searcher, queryConfiguration);
    LgteHits lgteHits = searcher.search(lgteQuery, queryFilter);

    assertEquals(lgteHits.length(), 1);
    assertEquals(lgteHits.id(0), 1);

    TermsFilter termsFilter = new TermsFilter();
    termsFilter.addTerm(new Term(Config.S_HAS_TIMEXES, "true"));
    lgteHits = searcher.search(lgteQuery, termsFilter);
    assertEquals(lgteHits.length(), 1);
    assertEquals(lgteHits.id(0), 1);

    termsFilter = new TermsFilter();
    termsFilter.addTerm(new Term(Config.S_HAS_TIMEXES + "_sentences", "true"));
    lgteHits = searcher.search(lgteQuery, termsFilter);
    assertEquals(lgteHits.length(), 1);
    assertEquals(lgteHits.id(0), 2);

    searcher.close();

    Files.delDirsE(path + "Contents");
    Files.delDirsE(path + "Sentences");
  }
Example #3
0
  protected void setUp() throws IOException {
    String contents1Doc1 =
        "word1 word2 word3 word32 word1 word45 word56 word67 word67 word2 word67 word88 word99 word99 word33";
    String contents2Doc1 = "word1 word2 word3 word32 word1 word45 word56 word67";
    String contents1Doc2 = "word2 word3 word4 word55 word96 word2 word54 word33 wordss";
    String contents2Doc2 = "word2 word3 word4 word55";
    String contents1Doc3 = "word1 word100 word400 word555 word966 word544 word333 wordss";
    String contents2Doc3 = "word1 word100";
    LgteIndexWriter writer = new LgteIndexWriter(pathUnique, true, Model.OkapiBM25Model);
    LgteDocumentWrapper doc1 = new LgteDocumentWrapper();
    doc1.indexText(Globals.DOCUMENT_ID_FIELD, "1");
    doc1.indexText("contents1", contents1Doc1);
    doc1.indexText("contents2", contents2Doc1);
    LgteDocumentWrapper doc2 = new LgteDocumentWrapper();
    doc2.indexText(Globals.DOCUMENT_ID_FIELD, "2");
    doc2.indexText("contents1", contents1Doc2);
    doc2.indexText("contents2", contents2Doc2);
    LgteDocumentWrapper doc3 = new LgteDocumentWrapper();
    doc3.indexText(Globals.DOCUMENT_ID_FIELD, "3");
    doc3.indexText("contents1", contents1Doc3);
    doc3.indexText("contents2", contents2Doc3);

    writer.addDocument(doc1);
    writer.addDocument(doc2);
    writer.addDocument(doc3);
    writer.close();

    // these two writers will replace the previous one
    // The first one will index contents1 and the second will index the contents2
    writer = new LgteIndexWriter(pathMulti1, true);
    doc1 = new LgteDocumentWrapper();
    doc1.indexText(Globals.DOCUMENT_ID_FIELD, "1");
    doc1.indexText("contents1", contents1Doc1);
    doc2 = new LgteDocumentWrapper();
    doc2.indexText(Globals.DOCUMENT_ID_FIELD, "2");
    doc2.indexText("contents1", contents1Doc2);
    doc3 = new LgteDocumentWrapper();
    doc3.indexText(Globals.DOCUMENT_ID_FIELD, "3");
    doc3.indexText("contents1", contents1Doc3);
    writer.addDocument(doc1);
    writer.addDocument(doc2);
    writer.addDocument(doc3);
    writer.close();

    writer = new LgteIndexWriter(pathMulti2, true);
    doc1 = new LgteDocumentWrapper();
    doc1.indexText(Globals.DOCUMENT_ID_FIELD, "1");
    doc1.indexText("contents2", contents2Doc1);
    doc2 = new LgteDocumentWrapper();
    doc2.indexText(Globals.DOCUMENT_ID_FIELD, "2");
    doc2.indexText("contents2", contents2Doc2);
    doc3 = new LgteDocumentWrapper();
    doc3.indexText(Globals.DOCUMENT_ID_FIELD, "3");
    doc3.indexText("contents2", contents2Doc3);
    writer.addDocument(doc1);
    writer.addDocument(doc2);
    writer.addDocument(doc3);
    writer.close();
  }
  public void testRange() throws IOException, InvalidGeoException {
    LgteIndexSearcherWrapper searcher;

    if (lm) searcher = new LgteIndexSearcherWrapper(Model.LanguageModel, path);
    else searcher = new LgteIndexSearcherWrapper(Model.VectorSpaceModel, path);

    int years = 14;

    // Building query
    QueryParams queryParams = new QueryParams();
    queryParams.setTime("1990-6-8");
    queryParams.setRadiumYears(years);
    // create a term level1query to searchCallback against indexText documents
    // doc is a word to find in text
    Query tq = new TermQuery(new Term("metafile", "doc"));

    LgteQuery lgteQuery = new LgteQuery(tq, queryParams);
    TimeDistanceSortSource dsort = new TimeDistanceSortSource();
    LgteSort sort = new LgteSort(new SortField("foo", dsort));

    LgteHits hits = searcher.search(lgteQuery, sort);
    int results = hits.length();

    // Get a list of distances, you don't need this but we keep it available, our LgteHits gives you
    // Distance Information
    ITimeDistancesWrapper timeDistancesWrapper = dsort.getTimeDistancesWrapper();

    // distances calculated from filter first pass must be less than total
    // docs, from the above test of 6 items, 5 will come from the boundary box
    // filter, but only 5 are actually in the radius of the results.

    // Note Boundary Box filtering, is not accurate enough for most systems.

    System.out.println(
        "Distance Filter filtered: " + timeDistancesWrapper.getTimeDistances().size());
    System.out.println("Results: " + results);
    System.out.println("=============================");
    assertEquals(5, timeDistancesWrapper.getTimeDistances().size());
    assertEquals(5, results);

    int lastYears = 0;
    long lastMili = 0;
    for (int i = 0; i < results; i++) {
      LgteDocumentWrapper d = hits.doc(i);

      String name = d.get("name");

      int distanceYears = hits.timeDistanceYears(i);
      long distanceMili = hits.timeDistanceMiliseconds(i);
      assertTrue(distanceYears <= years);
      assertTrue(distanceYears >= lastYears);
      assertTrue(distanceMili >= lastMili);
      lastYears = distanceYears;
      lastMili = distanceMili;

      System.out.println(
          "Name: " + name + ", Distance (years, mili):" + distanceYears + " |" + distanceMili);

      switch (i) {
        case 0:
          assertTrue(d.get("name").equals("1"));
          break;
        case 1:
          assertTrue(d.get("name").equals("2"));
          break;
        case 2:
          assertTrue(d.get("name").equals("3"));
          break;
        case 3:
          assertTrue(d.get("name").equals("4"));
          break;
        case 4:
          assertTrue(d.get("name").equals("5"));
          break;
      }
    }
    searcher.close();
  }