public void testLongFieldMinMax() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
int numDocs = atLeast(100);
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
long num = random().nextLong();
minValue = Math.min(num, minValue);
maxValue = Math.max(num, maxValue);
doc.add(new LegacyLongField("field", num, Field.Store.NO));
w.addDocument(doc);
}
IndexReader r = w.getReader();
Terms terms = MultiFields.getTerms(r, "field");
assertEquals(new Long(minValue), LegacyNumericUtils.getMinLong(terms));
assertEquals(new Long(maxValue), LegacyNumericUtils.getMaxLong(terms));
r.close();
w.close();
dir.close();
}
public void testMethod() throws Exception {
Directory directory = newDirectory();
String[] values = new String[] {"1", "2", "3", "4"};
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
for (int i = 0; i < values.length; i++) {
Document doc = new Document();
doc.add(newStringField(FIELD, values[i], Field.Store.YES));
writer.addDocument(doc);
}
IndexReader ir = writer.getReader();
writer.close();
BooleanQuery booleanQuery1 = new BooleanQuery();
booleanQuery1.add(new TermQuery(new Term(FIELD, "1")), BooleanClause.Occur.SHOULD);
booleanQuery1.add(new TermQuery(new Term(FIELD, "2")), BooleanClause.Occur.SHOULD);
BooleanQuery query = new BooleanQuery();
query.add(booleanQuery1, BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(FIELD, "9")), BooleanClause.Occur.MUST_NOT);
IndexSearcher indexSearcher = newSearcher(ir);
ScoreDoc[] hits = indexSearcher.search(query, null, 1000).scoreDocs;
assertEquals("Number of matched documents", 2, hits.length);
ir.close();
directory.close();
}
public void testDoubleFieldMinMax() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
int numDocs = atLeast(100);
double minValue = Double.POSITIVE_INFINITY;
double maxValue = Double.NEGATIVE_INFINITY;
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
double num = random().nextDouble();
minValue = Math.min(num, minValue);
maxValue = Math.max(num, maxValue);
doc.add(new LegacyDoubleField("field", num, Field.Store.NO));
w.addDocument(doc);
}
IndexReader r = w.getReader();
Terms terms = MultiFields.getTerms(r, "field");
assertEquals(
minValue, NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMinLong(terms)), 0.0);
assertEquals(
maxValue, NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMaxLong(terms)), 0.0);
r.close();
w.close();
dir.close();
}
@Override
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
RandomIndexWriter writer =
new RandomIndexWriter(
random(),
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMaxBufferedDocs(_TestUtil.nextInt(random(), 50, 1000)));
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.setOmitNorms(true);
Field field = newField("field", "", customType);
doc.add(field);
NumberFormat df = new DecimalFormat("000", new DecimalFormatSymbols(Locale.ROOT));
for (int i = 0; i < 1000; i++) {
field.setStringValue(df.format(i));
writer.addDocument(doc);
}
reader = writer.getReader();
writer.close();
searcher = newSearcher(reader);
}
public void testBasic() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
Document doc = new Document();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
Field f = newField("foo", "this is a test test", ft);
doc.add(f);
for (int i = 0; i < 100; i++) {
w.addDocument(doc);
}
IndexReader reader = w.getReader();
w.close();
assertNotNull(MultiFields.getTermPositionsEnum(reader, "foo", new BytesRef("test")));
PostingsEnum de =
TestUtil.docs(random(), reader, "foo", new BytesRef("test"), null, PostingsEnum.FREQS);
while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
assertEquals(2, de.freq());
}
reader.close();
dir.close();
}
@Override
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
RandomIndexWriter writer =
new RandomIndexWriter(
random(),
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy()));
// Add series of docs with filterable fields : url, text and dates flags
addDoc(writer, "http://lucene.apache.org", "lucene 1.4.3 available", "20040101");
addDoc(writer, "http://lucene.apache.org", "New release pending", "20040102");
addDoc(writer, "http://lucene.apache.org", "Lucene 1.9 out now", "20050101");
addDoc(writer, "http://www.bar.com", "Local man bites dog", "20040101");
addDoc(writer, "http://www.bar.com", "Dog bites local man", "20040102");
addDoc(writer, "http://www.bar.com", "Dog uses Lucene", "20050101");
addDoc(writer, "http://lucene.apache.org", "Lucene 2.0 out", "20050101");
addDoc(writer, "http://lucene.apache.org", "Oops. Lucene 2.1 out", "20050102");
// Until we fix LUCENE-2348, the index must
// have only 1 segment:
writer.forceMerge(1);
reader = writer.getReader();
writer.close();
searcher = newSearcher(reader);
}
public void testMoreThan32ProhibitedClauses() throws Exception {
final Directory d = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random(), d);
Document doc = new Document();
doc.add(
new TextField(
"field",
"0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33",
Field.Store.NO));
w.addDocument(doc);
doc = new Document();
doc.add(new TextField("field", "33", Field.Store.NO));
w.addDocument(doc);
final IndexReader r = w.getReader();
w.close();
final IndexSearcher s = newSearcher(r);
final BooleanQuery q = new BooleanQuery();
for (int term = 0; term < 33; term++) {
q.add(
new BooleanClause(
new TermQuery(new Term("field", "" + term)), BooleanClause.Occur.MUST_NOT));
}
q.add(new BooleanClause(new TermQuery(new Term("field", "33")), BooleanClause.Occur.SHOULD));
final int[] count = new int[1];
s.search(
q,
new Collector() {
private Scorer scorer;
@Override
public void setScorer(Scorer scorer) {
// Make sure we got BooleanScorer:
this.scorer = scorer;
assertEquals(
"Scorer is implemented by wrong class",
BooleanScorer.class.getName() + "$BucketScorer",
scorer.getClass().getName());
}
@Override
public void collect(int doc) {
count[0]++;
}
@Override
public void setNextReader(AtomicReaderContext context) {}
@Override
public boolean acceptsDocsOutOfOrder() {
return true;
}
});
assertEquals(1, count[0]);
r.close();
d.close();
}
public void testSetAllGroups() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w =
new RandomIndexWriter(
random(),
dir,
newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
Document doc = new Document();
doc.add(newField("group", "foo", StringField.TYPE_NOT_STORED));
doc.add(new SortedDocValuesField("group", new BytesRef("foo")));
w.addDocument(doc);
IndexSearcher indexSearcher = newSearcher(w.getReader());
w.close();
GroupingSearch gs = new GroupingSearch("group");
gs.setAllGroups(true);
TopGroups<?> groups = gs.search(indexSearcher, new TermQuery(new Term("group", "foo")), 0, 10);
assertEquals(1, groups.totalHitCount);
// assertEquals(1, groups.totalGroupCount.intValue());
assertEquals(1, groups.totalGroupedHitCount);
assertEquals(1, gs.getAllMatchingGroups().size());
indexSearcher.getIndexReader().close();
dir.close();
}
public void testNullDocIdSet() throws Exception {
// Tests that if a Filter produces a null DocIdSet, which is given to
// IndexSearcher, everything works fine. This came up in LUCENE-1754.
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("c", "val", Field.Store.NO));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
writer.close();
// First verify the document is searchable.
IndexSearcher searcher = newSearcher(reader);
Assert.assertEquals(1, searcher.search(new MatchAllDocsQuery(), 10).totalHits);
// Now search w/ a Filter which returns a null DocIdSet
Filter f =
new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
return null;
}
@Override
public String toString(String field) {
return "nullDocIdSetFilter";
}
};
Assert.assertEquals(
0, searcher.search(new FilteredQuery(new MatchAllDocsQuery(), f), 10).totalHits);
reader.close();
dir.close();
}
@Override
public void setUp() throws Exception {
super.setUp();
numIterations = atLeast(50);
dir = newDirectory();
RandomIndexWriter writer =
new RandomIndexWriter(
random(),
dir,
newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false))
.setMaxBufferedDocs(_TestUtil.nextInt(random(), 50, 1000)));
Document doc = new Document();
Field field = newStringField("field", "", Field.Store.YES);
doc.add(field);
terms = new TreeSet<BytesRef>();
int num = atLeast(200);
for (int i = 0; i < num; i++) {
String s = _TestUtil.randomUnicodeString(random());
field.setStringValue(s);
terms.add(new BytesRef(s));
writer.addDocument(doc);
}
termsAutomaton = BasicAutomata.makeStringUnion(terms);
reader = writer.getReader();
searcher = newSearcher(reader);
writer.close();
}
public void testEmptyDocs() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwConf =
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
// make sure that the fact that documents might be empty is not a problem
final Document emptyDoc = new Document();
final int numDocs = random().nextBoolean() ? 1 : atLeast(1000);
for (int i = 0; i < numDocs; ++i) {
iw.addDocument(emptyDoc);
}
iw.commit();
final DirectoryReader rd = DirectoryReader.open(dir);
for (int i = 0; i < numDocs; ++i) {
final Document doc = rd.document(i);
assertNotNull(doc);
assertTrue(doc.getFields().isEmpty());
}
rd.close();
iw.close();
dir.close();
}
@Test
public void testMultipleSegments() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
int num = Math.min(1000, atLeast(10));
List<Entry> entries = new ArrayList<>();
// ensure at least some segments have no suggest field
for (int i = num; i > 0; i--) {
Document document = new Document();
if (random().nextInt(4) == 1) {
document.add(new SuggestField("suggest_field", "abc_" + i, i));
entries.add(new Entry("abc_" + i, i));
}
document.add(new StoredField("weight_fld", i));
iw.addDocument(document);
if (usually()) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
TopSuggestDocs suggest =
indexSearcher.suggest(query, (entries.size() == 0) ? 1 : entries.size());
assertSuggestions(suggest, entries.toArray(new Entry[entries.size()]));
reader.close();
iw.close();
}
public void testMax() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new SortedSetDocValuesField("value", new BytesRef("foo")));
doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
doc.add(newStringField("id", "2", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
// slow wrapper does not support random access ordinals (there is no need for that!)
IndexSearcher searcher = newSearcher(ir, false);
Sort sort = new Sort(new SortedSetSortField("value", false, SortedSetSortField.Selector.MAX));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(2, td.totalHits);
// 'baz' comes before 'foo'
assertEquals("2", searcher.doc(td.scoreDocs[0].doc).get("id"));
assertEquals("1", searcher.doc(td.scoreDocs[1].doc).get("id"));
assertNoFieldCaches();
ir.close();
dir.close();
}
@Test
public void testEarlyTermination() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
int num = Math.min(1000, atLeast(10));
// have segments of 4 documents
// with descending suggestion weights
// suggest should early terminate for
// segments with docs having lower suggestion weights
for (int i = num; i > 0; i--) {
Document document = new Document();
document.add(new SuggestField("suggest_field", "abc_" + i, i));
iw.addDocument(document);
if (i % 4 == 0) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
TopSuggestDocs suggest = indexSearcher.suggest(query, 1);
assertSuggestions(suggest, new Entry("abc_" + num, num));
reader.close();
iw.close();
}
@Test
public void testSuggestOnAllFilteredDocuments() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
int num = Math.min(1000, atLeast(10));
for (int i = 0; i < num; i++) {
Document document = new Document();
document.add(new SuggestField("suggest_field", "abc_" + i, i));
document.add(newStringField("str_fld", "deleted", Field.Store.NO));
iw.addDocument(document);
if (usually()) {
iw.commit();
}
}
BitsProducer filter =
new BitsProducer() {
@Override
public Bits getBits(LeafReaderContext context) throws IOException {
return new Bits.MatchNoBits(context.reader().maxDoc());
}
};
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
// no random access required;
// calling suggest with filter that does not match any documents should early terminate
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"), filter);
TopSuggestDocs suggest = indexSearcher.suggest(query, num);
assertThat(suggest.totalHits, equalTo(0));
reader.close();
iw.close();
}
@Test
public void testDupSuggestFieldValues() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
final int num = Math.min(1000, atLeast(300));
int[] weights = new int[num];
for (int i = 0; i < num; i++) {
Document document = new Document();
weights[i] = Math.abs(random().nextInt());
document.add(new SuggestField("suggest_field", "abc", weights[i]));
iw.addDocument(document);
if (usually()) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
Entry[] expectedEntries = new Entry[num];
Arrays.sort(weights);
for (int i = 1; i <= num; i++) {
expectedEntries[i - 1] = new Entry("abc", weights[num - i]);
}
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc"));
TopSuggestDocs lookupDocs = suggestIndexSearcher.suggest(query, num);
assertSuggestions(lookupDocs, expectedEntries);
reader.close();
iw.close();
}
@Test
public void testContextPrecedenceBoost() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
Document document = new Document();
document.add(new ContextSuggestField("suggest_field", "suggestion1", 4, "typetype"));
document.add(new ContextSuggestField("suggest_field", "suggestion2", 3, "type"));
iw.addDocument(document);
if (rarely()) {
iw.commit();
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
ContextQuery query =
new ContextQuery(new PrefixCompletionQuery(analyzer, new Term("suggest_field", "sugg")));
query.addContext("type", 1);
query.addContext("typetype", 2);
TopSuggestDocs suggest = suggestIndexSearcher.suggest(query, 5);
assertSuggestions(
suggest,
new Entry("suggestion1", "typetype", 4 * 2),
new Entry("suggestion2", "type", 3 * 1));
reader.close();
iw.close();
}
private IndexSearcher getSearcher() throws Exception {
directory = newDirectory();
String[] docs =
new String[] {
"xx rr yy mm pp",
"xx yy mm rr pp",
"nopayload qq ss pp np",
"one two three four five six seven eight nine ten eleven",
"nine one two three four five six seven eight eleven ten"
};
RandomIndexWriter writer =
new RandomIndexWriter(
random(),
directory,
newIndexWriterConfig(new PayloadAnalyzer()).setSimilarity(similarity));
Document doc = null;
for (int i = 0; i < docs.length; i++) {
doc = new Document();
String docText = docs[i];
doc.add(newTextField(PayloadHelper.FIELD, docText, Field.Store.YES));
writer.addDocument(doc);
}
closeIndexReader = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(closeIndexReader);
return searcher;
}
private float checkPhraseQuery(Document doc, PhraseQuery query, int slop, int expectedNumResults)
throws Exception {
query.setSlop(slop);
Directory ramDir = newDirectory();
RandomIndexWriter writer =
new RandomIndexWriter(random, ramDir, new MockAnalyzer(MockTokenizer.WHITESPACE, false));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
TopDocs td = searcher.search(query, null, 10);
// System.out.println("slop: "+slop+" query: "+query+" doc: "+doc+" Expecting number of hits:
// "+expectedNumResults+" maxScore="+td.getMaxScore());
assertEquals(
"slop: " + slop + " query: " + query + " doc: " + doc + " Wrong number of hits",
expectedNumResults,
td.totalHits);
// QueryUtils.check(query,searcher);
writer.close();
searcher.close();
reader.close();
ramDir.close();
return td.getMaxScore();
}
public void testSpanNot() throws Exception {
SpanQuery[] clauses = new SpanQuery[2];
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "three"));
SpanQuery spq = new SpanNearQuery(clauses, 5, true);
SpanNotQuery snq =
new SpanNotQuery(spq, new SpanTermQuery(new Term(PayloadHelper.FIELD, "two")));
Directory directory = newDirectory();
RandomIndexWriter writer =
new RandomIndexWriter(
random(),
directory,
newIndexWriterConfig(new PayloadAnalyzer()).setSimilarity(similarity));
Document doc = new Document();
doc.add(newTextField(PayloadHelper.FIELD, "one two three one four three", Field.Store.YES));
writer.addDocument(doc);
IndexReader reader = writer.getReader();
writer.close();
checkSpans(MultiSpansWrapper.wrap(reader, snq, SpanWeight.Postings.PAYLOADS), 1, new int[] {2});
reader.close();
directory.close();
}
public void testCachingWorks() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
writer.close();
IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
MockFilter filter = new MockFilter();
CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
// first time, nested filter is called
DocIdSet strongRef = cacher.getDocIdSet(context, context.reader().getLiveDocs());
assertTrue("first time", filter.wasCalled());
// make sure no exception if cache is holding the wrong docIdSet
cacher.getDocIdSet(context, context.reader().getLiveDocs());
// second time, nested filter should not be called
filter.clear();
cacher.getDocIdSet(context, context.reader().getLiveDocs());
assertFalse("second time", filter.wasCalled());
reader.close();
dir.close();
}
public void testNullDocIdSetIterator() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
writer.close();
IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
final Filter filter =
new Filter() {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
return new DocIdSet() {
@Override
public DocIdSetIterator iterator() {
return null;
}
};
}
};
CachingWrapperFilter cacher = new CachingWrapperFilter(filter);
// the caching filter should return the empty set constant
assertNull(cacher.getDocIdSet(context, context.reader().getLiveDocs()));
reader.close();
dir.close();
}
public void testEvilSearcherFactory() throws Exception {
final Directory dir = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
w.commit();
final IndexReader other = DirectoryReader.open(dir);
final SearcherFactory theEvilOne =
new SearcherFactory() {
@Override
public IndexSearcher newSearcher(IndexReader ignored) {
return LuceneTestCase.newSearcher(other);
}
};
try {
new SearcherManager(w.w, false, theEvilOne);
fail("didn't hit expected exception");
} catch (IllegalStateException ise) {
// expected
}
w.close();
other.close();
dir.close();
}
public void test() throws Exception {
BaseDirectoryWrapper d = newDirectory();
d.setCheckIndexOnClose(false);
// we nuke files, but verify the reader still works
RandomIndexWriter w = new RandomIndexWriter(random(), d);
int numDocs = atLeast(100);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
doc.add(newField("foo", "bar", TextField.TYPE_NOT_STORED));
w.addDocument(doc);
}
IndexReader r = w.getReader();
w.commit();
w.close();
for (String fileName : d.listAll()) {
try {
d.deleteFile(fileName);
// may succeed, e.g. if the file is completely read into RAM.
} catch (IOException ioe) {
// ignore: this means codec (correctly) is holding
// the file open
}
}
for (LeafReaderContext cxt : r.leaves()) {
TestUtil.checkReader(cxt.reader());
}
r.close();
d.close();
}
public void testReuseDocsEnumNoReuse() throws IOException {
Directory dir = newDirectory();
Codec cp = TestUtil.alwaysPostingsFormat(new Lucene40RWPostingsFormat());
RandomIndexWriter writer =
new RandomIndexWriter(
random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setCodec(cp));
int numdocs = atLeast(20);
createRandomIndex(numdocs, writer, random());
writer.commit();
DirectoryReader open = DirectoryReader.open(dir);
for (LeafReaderContext ctx : open.leaves()) {
LeafReader indexReader = ctx.reader();
Terms terms = indexReader.terms("body");
TermsEnum iterator = terms.iterator();
IdentityHashMap<PostingsEnum, Boolean> enums = new IdentityHashMap<>();
MatchNoBits bits = new Bits.MatchNoBits(indexReader.maxDoc());
while ((iterator.next()) != null) {
PostingsEnum docs =
iterator.postings(
random().nextBoolean() ? bits : new Bits.MatchNoBits(indexReader.maxDoc()),
null,
random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(terms.size(), enums.size());
}
writer.commit();
IOUtils.close(writer, open, dir);
}
public void testNRTAndCommit() throws Exception {
Directory dir = newDirectory();
NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir, 2.0, 25.0);
MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
IndexWriterConfig conf = newIndexWriterConfig(analyzer);
RandomIndexWriter w = new RandomIndexWriter(random(), cachedDir, conf);
final LineFileDocs docs = new LineFileDocs(random(), true);
final int numDocs = TestUtil.nextInt(random(), 100, 400);
if (VERBOSE) {
System.out.println("TEST: numDocs=" + numDocs);
}
final List<BytesRef> ids = new ArrayList<>();
DirectoryReader r = null;
for (int docCount = 0; docCount < numDocs; docCount++) {
final Document doc = docs.nextDoc();
ids.add(new BytesRef(doc.get("docid")));
w.addDocument(doc);
if (random().nextInt(20) == 17) {
if (r == null) {
r = DirectoryReader.open(w.w);
} else {
final DirectoryReader r2 = DirectoryReader.openIfChanged(r);
if (r2 != null) {
r.close();
r = r2;
}
}
assertEquals(1 + docCount, r.numDocs());
final IndexSearcher s = newSearcher(r);
// Just make sure search can run; we can't assert
// totHits since it could be 0
TopDocs hits = s.search(new TermQuery(new Term("body", "the")), 10);
// System.out.println("tot hits " + hits.totalHits);
}
}
if (r != null) {
r.close();
}
// Close should force cache to clear since all files are sync'd
w.close();
final String[] cachedFiles = cachedDir.listCachedFiles();
for (String file : cachedFiles) {
System.out.println("FAIL: cached file " + file + " remains after sync");
}
assertEquals(0, cachedFiles.length);
r = DirectoryReader.open(dir);
for (BytesRef id : ids) {
assertEquals(1, r.docFreq(new Term("docid", id)));
}
r.close();
cachedDir.close();
docs.close();
}
@Test
public void testRandomContextQueryScoring() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
try (RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"))) {
int numSuggestions = atLeast(20);
int numContexts = atLeast(5);
Set<Integer> seenWeights = new HashSet<>();
List<Entry> expectedEntries = new ArrayList<>();
List<CharSequence> contexts = new ArrayList<>();
for (int i = 1; i <= numContexts; i++) {
CharSequence context = TestUtil.randomSimpleString(random(), 10) + i;
contexts.add(context);
for (int j = 1; j <= numSuggestions; j++) {
String suggestion = "sugg_" + TestUtil.randomSimpleString(random(), 10) + j;
int weight = TestUtil.nextInt(random(), 1, 1000 * numContexts * numSuggestions);
while (seenWeights.contains(weight)) {
weight = TestUtil.nextInt(random(), 1, 1000 * numContexts * numSuggestions);
}
seenWeights.add(weight);
Document document = new Document();
document.add(new ContextSuggestField("suggest_field", suggestion, weight, context));
iw.addDocument(document);
expectedEntries.add(new Entry(suggestion, context.toString(), i * weight));
}
if (rarely()) {
iw.commit();
}
}
Entry[] expectedResults = expectedEntries.toArray(new Entry[expectedEntries.size()]);
ArrayUtil.introSort(
expectedResults,
new Comparator<Entry>() {
@Override
public int compare(Entry o1, Entry o2) {
int cmp = Float.compare(o2.value, o1.value);
if (cmp != 0) {
return cmp;
} else {
return o1.output.compareTo(o2.output);
}
}
});
try (DirectoryReader reader = iw.getReader()) {
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
ContextQuery query =
new ContextQuery(
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "sugg")));
for (int i = 0; i < contexts.size(); i++) {
query.addContext(contexts.get(i), i + 1);
}
TopSuggestDocs suggest = suggestIndexSearcher.suggest(query, 4);
assertSuggestions(suggest, Arrays.copyOfRange(expectedResults, 0, 4));
}
}
}
// LUCENE-2095: make sure with multiple threads commit
// doesn't return until all changes are in fact in the
// index
public void testCommitThreadSafety() throws Throwable {
final int NUM_THREADS = 5;
final double RUN_SEC = 0.5;
final Directory dir = newDirectory();
final RandomIndexWriter w =
new RandomIndexWriter(
random(),
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy()));
_TestUtil.reduceOpenFiles(w.w);
w.commit();
final AtomicBoolean failed = new AtomicBoolean();
Thread[] threads = new Thread[NUM_THREADS];
final long endTime = System.currentTimeMillis() + ((long) (RUN_SEC * 1000));
for (int i = 0; i < NUM_THREADS; i++) {
final int finalI = i;
threads[i] =
new Thread() {
@Override
public void run() {
try {
final Document doc = new Document();
DirectoryReader r = DirectoryReader.open(dir);
Field f = newStringField("f", "", Field.Store.NO);
doc.add(f);
int count = 0;
do {
if (failed.get()) break;
for (int j = 0; j < 10; j++) {
final String s = finalI + "_" + String.valueOf(count++);
f.setStringValue(s);
w.addDocument(doc);
w.commit();
DirectoryReader r2 = DirectoryReader.openIfChanged(r);
assertNotNull(r2);
assertTrue(r2 != r);
r.close();
r = r2;
assertEquals("term=f:" + s + "; r=" + r, 1, r.docFreq(new Term("f", s)));
}
} while (System.currentTimeMillis() < endTime);
r.close();
} catch (Throwable t) {
failed.set(true);
throw new RuntimeException(t);
}
}
};
threads[i].start();
}
for (int i = 0; i < NUM_THREADS; i++) {
threads[i].join();
}
assertFalse(failed.get());
w.close();
dir.close();
}
@BeforeClass
public static void beforeClassDrillDownQueryTest() throws Exception {
dir = newDirectory();
Random r = random();
RandomIndexWriter writer =
new RandomIndexWriter(
r,
dir,
newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(r, MockTokenizer.KEYWORD, false)));
taxoDir = newDirectory();
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
config = new FacetsConfig();
// Randomize the per-dim config:
config.setHierarchical("a", random().nextBoolean());
config.setMultiValued("a", random().nextBoolean());
if (random().nextBoolean()) {
config.setIndexFieldName("a", "$a");
}
config.setRequireDimCount("a", true);
config.setHierarchical("b", random().nextBoolean());
config.setMultiValued("b", random().nextBoolean());
if (random().nextBoolean()) {
config.setIndexFieldName("b", "$b");
}
config.setRequireDimCount("b", true);
for (int i = 0; i < 100; i++) {
Document doc = new Document();
if (i % 2 == 0) { // 50
doc.add(new TextField("content", "foo", Field.Store.NO));
}
if (i % 3 == 0) { // 33
doc.add(new TextField("content", "bar", Field.Store.NO));
}
if (i % 4 == 0) { // 25
if (r.nextBoolean()) {
doc.add(new FacetField("a", "1"));
} else {
doc.add(new FacetField("a", "2"));
}
}
if (i % 5 == 0) { // 20
doc.add(new FacetField("b", "1"));
}
writer.addDocument(config.build(taxoWriter, doc));
}
taxoWriter.close();
reader = writer.getReader();
writer.close();
taxo = new DirectoryTaxonomyReader(taxoDir);
}
@BeforeClass
public static void beforeClass() throws Exception {
directory = newDirectory();
RandomIndexWriter writer =
new RandomIndexWriter(
random,
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))
.setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)));
NumericField field8 = new NumericField("field8", 8, Field.Store.YES, true),
field6 = new NumericField("field6", 6, Field.Store.YES, true),
field4 = new NumericField("field4", 4, Field.Store.YES, true),
field2 = new NumericField("field2", 2, Field.Store.YES, true),
fieldNoTrie =
new NumericField("field" + Integer.MAX_VALUE, Integer.MAX_VALUE, Field.Store.YES, true),
ascfield8 = new NumericField("ascfield8", 8, Field.Store.NO, true),
ascfield6 = new NumericField("ascfield6", 6, Field.Store.NO, true),
ascfield4 = new NumericField("ascfield4", 4, Field.Store.NO, true),
ascfield2 = new NumericField("ascfield2", 2, Field.Store.NO, true);
Document doc = new Document();
// add fields, that have a distance to test general functionality
doc.add(field8);
doc.add(field6);
doc.add(field4);
doc.add(field2);
doc.add(fieldNoTrie);
// add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct
// splitting of range and inclusive/exclusive
doc.add(ascfield8);
doc.add(ascfield6);
doc.add(ascfield4);
doc.add(ascfield2);
// Add a series of noDocs docs with increasing long values, by updating the fields
for (int l = 0; l < noDocs; l++) {
long val = distance * l + startOffset;
field8.setLongValue(val);
field6.setLongValue(val);
field4.setLongValue(val);
field2.setLongValue(val);
fieldNoTrie.setLongValue(val);
val = l - (noDocs / 2);
ascfield8.setLongValue(val);
ascfield6.setLongValue(val);
ascfield4.setLongValue(val);
ascfield2.setLongValue(val);
writer.addDocument(doc);
}
reader = writer.getReader();
searcher = newSearcher(reader);
writer.close();
}