public void testReuseDocsEnumNoReuse() throws IOException {
Directory dir = newDirectory();
Codec cp = TestUtil.alwaysPostingsFormat(new Lucene40RWPostingsFormat());
RandomIndexWriter writer =
new RandomIndexWriter(
random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setCodec(cp));
int numdocs = atLeast(20);
createRandomIndex(numdocs, writer, random());
writer.commit();
DirectoryReader open = DirectoryReader.open(dir);
for (LeafReaderContext ctx : open.leaves()) {
LeafReader indexReader = ctx.reader();
Terms terms = indexReader.terms("body");
TermsEnum iterator = terms.iterator();
IdentityHashMap<PostingsEnum, Boolean> enums = new IdentityHashMap<>();
MatchNoBits bits = new Bits.MatchNoBits(indexReader.maxDoc());
while ((iterator.next()) != null) {
PostingsEnum docs =
iterator.postings(
random().nextBoolean() ? bits : new Bits.MatchNoBits(indexReader.maxDoc()),
null,
random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(terms.size(), enums.size());
}
writer.commit();
IOUtils.close(writer, open, dir);
}
public void testEmptyDocs() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwConf =
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
// make sure that the fact that documents might be empty is not a problem
final Document emptyDoc = new Document();
final int numDocs = random().nextBoolean() ? 1 : atLeast(1000);
for (int i = 0; i < numDocs; ++i) {
iw.addDocument(emptyDoc);
}
iw.commit();
final DirectoryReader rd = DirectoryReader.open(dir);
for (int i = 0; i < numDocs; ++i) {
final Document doc = rd.document(i);
assertNotNull(doc);
assertTrue(doc.getFields().isEmpty());
}
rd.close();
iw.close();
dir.close();
}
@Test
public void testContextQueryOnSuggestField() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
Document document = new Document();
document.add(new SuggestField("suggest_field", "abc", 3));
document.add(new SuggestField("suggest_field", "abd", 4));
document.add(new SuggestField("suggest_field", "The Foo Fighters", 2));
iw.addDocument(document);
document = new Document();
document.add(new SuggestField("suggest_field", "abcdd", 5));
iw.addDocument(document);
if (rarely()) {
iw.commit();
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
ContextQuery query =
new ContextQuery(new PrefixCompletionQuery(analyzer, new Term("suggest_field", "ab")));
try {
suggestIndexSearcher.suggest(query, 4);
} catch (IllegalStateException expected) {
assertTrue(expected.getMessage().contains("SuggestField"));
}
reader.close();
iw.close();
}
@Test
public void testAllContextQuery() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
Document document = new Document();
document.add(new ContextSuggestField("suggest_field", "suggestion1", 4, "type1"));
document.add(new ContextSuggestField("suggest_field", "suggestion2", 3, "type2"));
document.add(new ContextSuggestField("suggest_field", "suggestion3", 2, "type3"));
iw.addDocument(document);
document = new Document();
document.add(new ContextSuggestField("suggest_field", "suggestion4", 1, "type4"));
iw.addDocument(document);
if (rarely()) {
iw.commit();
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
ContextQuery query =
new ContextQuery(new PrefixCompletionQuery(analyzer, new Term("suggest_field", "sugg")));
TopSuggestDocs suggest = suggestIndexSearcher.suggest(query, 4);
assertSuggestions(
suggest,
new Entry("suggestion1", "type1", 4),
new Entry("suggestion2", "type2", 3),
new Entry("suggestion3", "type3", 2),
new Entry("suggestion4", "type4", 1));
reader.close();
iw.close();
}
@Test
public void testMultipleSegments() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
int num = Math.min(1000, atLeast(10));
List<Entry> entries = new ArrayList<>();
// ensure at least some segments have no suggest field
for (int i = num; i > 0; i--) {
Document document = new Document();
if (random().nextInt(4) == 1) {
document.add(new SuggestField("suggest_field", "abc_" + i, i));
entries.add(new Entry("abc_" + i, i));
}
document.add(new StoredField("weight_fld", i));
iw.addDocument(document);
if (usually()) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
TopSuggestDocs suggest =
indexSearcher.suggest(query, (entries.size() == 0) ? 1 : entries.size());
assertSuggestions(suggest, entries.toArray(new Entry[entries.size()]));
reader.close();
iw.close();
}
@Test
public void testEarlyTermination() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
int num = Math.min(1000, atLeast(10));
// have segments of 4 documents
// with descending suggestion weights
// suggest should early terminate for
// segments with docs having lower suggestion weights
for (int i = num; i > 0; i--) {
Document document = new Document();
document.add(new SuggestField("suggest_field", "abc_" + i, i));
iw.addDocument(document);
if (i % 4 == 0) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
TopSuggestDocs suggest = indexSearcher.suggest(query, 1);
assertSuggestions(suggest, new Entry("abc_" + num, num));
reader.close();
iw.close();
}
@Test
public void testDupSuggestFieldValues() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
final int num = Math.min(1000, atLeast(300));
int[] weights = new int[num];
for (int i = 0; i < num; i++) {
Document document = new Document();
weights[i] = Math.abs(random().nextInt());
document.add(new SuggestField("suggest_field", "abc", weights[i]));
iw.addDocument(document);
if (usually()) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
Entry[] expectedEntries = new Entry[num];
Arrays.sort(weights);
for (int i = 1; i <= num; i++) {
expectedEntries[i - 1] = new Entry("abc", weights[num - i]);
}
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc"));
TopSuggestDocs lookupDocs = suggestIndexSearcher.suggest(query, num);
assertSuggestions(lookupDocs, expectedEntries);
reader.close();
iw.close();
}
@Test
public void testSuggestOnAllFilteredDocuments() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
int num = Math.min(1000, atLeast(10));
for (int i = 0; i < num; i++) {
Document document = new Document();
document.add(new SuggestField("suggest_field", "abc_" + i, i));
document.add(newStringField("str_fld", "deleted", Field.Store.NO));
iw.addDocument(document);
if (usually()) {
iw.commit();
}
}
BitsProducer filter =
new BitsProducer() {
@Override
public Bits getBits(LeafReaderContext context) throws IOException {
return new Bits.MatchNoBits(context.reader().maxDoc());
}
};
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
// no random access required;
// calling suggest with filter that does not match any documents should early terminate
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"), filter);
TopSuggestDocs suggest = indexSearcher.suggest(query, num);
assertThat(suggest.totalHits, equalTo(0));
reader.close();
iw.close();
}
public void test() throws Exception {
BaseDirectoryWrapper d = newDirectory();
d.setCheckIndexOnClose(false);
// we nuke files, but verify the reader still works
RandomIndexWriter w = new RandomIndexWriter(random(), d);
int numDocs = atLeast(100);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
doc.add(newField("foo", "bar", TextField.TYPE_NOT_STORED));
w.addDocument(doc);
}
IndexReader r = w.getReader();
w.commit();
w.close();
for (String fileName : d.listAll()) {
try {
d.deleteFile(fileName);
// may succeed, e.g. if the file is completely read into RAM.
} catch (IOException ioe) {
// ignore: this means codec (correctly) is holding
// the file open
}
}
for (LeafReaderContext cxt : r.leaves()) {
TestUtil.checkReader(cxt.reader());
}
r.close();
d.close();
}
public void testEvilSearcherFactory() throws Exception {
final Directory dir = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
w.commit();
final IndexReader other = DirectoryReader.open(dir);
final SearcherFactory theEvilOne =
new SearcherFactory() {
@Override
public IndexSearcher newSearcher(IndexReader ignored) {
return LuceneTestCase.newSearcher(other);
}
};
try {
new SearcherManager(w.w, false, theEvilOne);
fail("didn't hit expected exception");
} catch (IllegalStateException ise) {
// expected
}
w.close();
other.close();
dir.close();
}
@Test
public void testFuzzyQuery() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
Document document = new Document();
document.add(new SuggestField("suggest_field", "suggestion", 2));
document.add(new SuggestField("suggest_field", "suaggestion", 4));
document.add(new SuggestField("suggest_field", "ssuggestion", 1));
iw.addDocument(document);
document = new Document();
document.add(new SuggestField("suggest_field", "sugfoo", 1));
iw.addDocument(document);
if (rarely()) {
iw.commit();
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
CompletionQuery query = new FuzzyCompletionQuery(analyzer, new Term("suggest_field", "sugg"));
TopSuggestDocs suggest = suggestIndexSearcher.suggest(query, 4);
assertSuggestions(
suggest,
new Entry("suaggestion", 4 * 2),
new Entry("suggestion", 2 * 3),
new Entry("sugfoo", 1 * 3),
new Entry("ssuggestion", 1 * 1));
reader.close();
iw.close();
}
/** make sure we downgrade positions and payloads correctly */
public void testMixing() throws Exception {
// no positions
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
for (int i = 0; i < 20; i++) {
Document doc = new Document();
if (i < 19 && random().nextBoolean()) {
for (int j = 0; j < 50; j++) {
doc.add(new TextField("foo", "i have positions", Field.Store.NO));
}
} else {
for (int j = 0; j < 50; j++) {
doc.add(new Field("foo", "i have no positions", ft));
}
}
iw.addDocument(doc);
iw.commit();
}
if (random().nextBoolean()) {
iw.forceMerge(1);
}
DirectoryReader ir = iw.getReader();
FieldInfos fis = MultiFields.getMergedFieldInfos(ir);
assertEquals(IndexOptions.DOCS_AND_FREQS, fis.fieldInfo("foo").getIndexOptions());
assertFalse(fis.fieldInfo("foo").hasPayloads());
iw.close();
ir.close();
dir.close(); // checkindex
}
public void testNumerics() throws Exception {
Directory dir = newDirectory();
Document doc = new Document();
Field field = new NumericDocValuesField("numbers", 0);
doc.add(field);
IndexWriterConfig iwc = newIndexWriterConfig(random(), null);
iwc.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
int numDocs = TEST_NIGHTLY ? atLeast(500) : atLeast(50);
for (int i = 0; i < numDocs; i++) {
field.setLongValue(random().nextLong());
iw.addDocument(doc);
if (random().nextInt(17) == 0) {
iw.commit();
}
}
DirectoryReader ir = iw.getReader();
iw.forceMerge(1);
DirectoryReader ir2 = iw.getReader();
LeafReader merged = getOnlyLeafReader(ir2);
iw.close();
NumericDocValues multi = MultiDocValues.getNumericValues(ir, "numbers");
NumericDocValues single = merged.getNumericDocValues("numbers");
for (int i = 0; i < numDocs; i++) {
assertEquals(single.get(i), multi.get(i));
}
ir.close();
ir2.close();
dir.close();
}
@Test
public void testRandomContextQueryScoring() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
try (RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"))) {
int numSuggestions = atLeast(20);
int numContexts = atLeast(5);
Set<Integer> seenWeights = new HashSet<>();
List<Entry> expectedEntries = new ArrayList<>();
List<CharSequence> contexts = new ArrayList<>();
for (int i = 1; i <= numContexts; i++) {
CharSequence context = TestUtil.randomSimpleString(random(), 10) + i;
contexts.add(context);
for (int j = 1; j <= numSuggestions; j++) {
String suggestion = "sugg_" + TestUtil.randomSimpleString(random(), 10) + j;
int weight = TestUtil.nextInt(random(), 1, 1000 * numContexts * numSuggestions);
while (seenWeights.contains(weight)) {
weight = TestUtil.nextInt(random(), 1, 1000 * numContexts * numSuggestions);
}
seenWeights.add(weight);
Document document = new Document();
document.add(new ContextSuggestField("suggest_field", suggestion, weight, context));
iw.addDocument(document);
expectedEntries.add(new Entry(suggestion, context.toString(), i * weight));
}
if (rarely()) {
iw.commit();
}
}
Entry[] expectedResults = expectedEntries.toArray(new Entry[expectedEntries.size()]);
ArrayUtil.introSort(
expectedResults,
new Comparator<Entry>() {
@Override
public int compare(Entry o1, Entry o2) {
int cmp = Float.compare(o2.value, o1.value);
if (cmp != 0) {
return cmp;
} else {
return o1.output.compareTo(o2.output);
}
}
});
try (DirectoryReader reader = iw.getReader()) {
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
ContextQuery query =
new ContextQuery(
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "sugg")));
for (int i = 0; i < contexts.size(); i++) {
query.addContext(contexts.get(i), i + 1);
}
TopSuggestDocs suggest = suggestIndexSearcher.suggest(query, 4);
assertSuggestions(suggest, Arrays.copyOfRange(expectedResults, 0, 4));
}
}
}
// LUCENE-2095: make sure with multiple threads commit
// doesn't return until all changes are in fact in the
// index
public void testCommitThreadSafety() throws Throwable {
final int NUM_THREADS = 5;
final double RUN_SEC = 0.5;
final Directory dir = newDirectory();
final RandomIndexWriter w =
new RandomIndexWriter(
random(),
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy()));
_TestUtil.reduceOpenFiles(w.w);
w.commit();
final AtomicBoolean failed = new AtomicBoolean();
Thread[] threads = new Thread[NUM_THREADS];
final long endTime = System.currentTimeMillis() + ((long) (RUN_SEC * 1000));
for (int i = 0; i < NUM_THREADS; i++) {
final int finalI = i;
threads[i] =
new Thread() {
@Override
public void run() {
try {
final Document doc = new Document();
DirectoryReader r = DirectoryReader.open(dir);
Field f = newStringField("f", "", Field.Store.NO);
doc.add(f);
int count = 0;
do {
if (failed.get()) break;
for (int j = 0; j < 10; j++) {
final String s = finalI + "_" + String.valueOf(count++);
f.setStringValue(s);
w.addDocument(doc);
w.commit();
DirectoryReader r2 = DirectoryReader.openIfChanged(r);
assertNotNull(r2);
assertTrue(r2 != r);
r.close();
r = r2;
assertEquals("term=f:" + s + "; r=" + r, 1, r.docFreq(new Term("f", s)));
}
} while (System.currentTimeMillis() < endTime);
r.close();
} catch (Throwable t) {
failed.set(true);
throw new RuntimeException(t);
}
}
};
threads[i].start();
}
for (int i = 0; i < NUM_THREADS; i++) {
threads[i].join();
}
assertFalse(failed.get());
w.close();
dir.close();
}
protected void commit() throws IOException {
indexWriter.commit();
DirectoryReader newReader = DirectoryReader.openIfChanged(indexReader);
if (newReader != null) {
IOUtils.close(indexReader);
indexReader = newReader;
}
indexSearcher = newSearcher(indexReader);
}
public void testReadSkip() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwConf =
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
FieldType ft = new FieldType();
ft.setStored(true);
ft.freeze();
final String string = _TestUtil.randomSimpleString(random(), 50);
final byte[] bytes = string.getBytes("UTF-8");
final long l = random().nextBoolean() ? random().nextInt(42) : random().nextLong();
final int i = random().nextBoolean() ? random().nextInt(42) : random().nextInt();
final float f = random().nextFloat();
final double d = random().nextDouble();
List<Field> fields =
Arrays.asList(
new Field("bytes", bytes, ft),
new Field("string", string, ft),
new LongField("long", l, Store.YES),
new IntField("int", i, Store.YES),
new FloatField("float", f, Store.YES),
new DoubleField("double", d, Store.YES));
for (int k = 0; k < 100; ++k) {
Document doc = new Document();
for (Field fld : fields) {
doc.add(fld);
}
iw.w.addDocument(doc);
}
iw.commit();
final DirectoryReader reader = DirectoryReader.open(dir);
final int docID = random().nextInt(100);
for (Field fld : fields) {
String fldName = fld.name();
final Document sDoc = reader.document(docID, Collections.singleton(fldName));
final IndexableField sField = sDoc.getField(fldName);
if (Field.class.equals(fld.getClass())) {
assertEquals(fld.binaryValue(), sField.binaryValue());
assertEquals(fld.stringValue(), sField.stringValue());
} else {
assertEquals(fld.numericValue(), sField.numericValue());
}
}
reader.close();
iw.close();
dir.close();
}
public void testSortedNumeric() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(random(), null);
iwc.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
int numDocs = TEST_NIGHTLY ? atLeast(500) : atLeast(50);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
int numValues = random().nextInt(5);
for (int j = 0; j < numValues; j++) {
doc.add(
new SortedNumericDocValuesField(
"nums", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
}
iw.addDocument(doc);
if (random().nextInt(17) == 0) {
iw.commit();
}
}
DirectoryReader ir = iw.getReader();
iw.forceMerge(1);
DirectoryReader ir2 = iw.getReader();
LeafReader merged = getOnlyLeafReader(ir2);
iw.close();
SortedNumericDocValues multi = MultiDocValues.getSortedNumericValues(ir, "nums");
SortedNumericDocValues single = merged.getSortedNumericDocValues("nums");
if (multi == null) {
assertNull(single);
} else {
// check values
for (int i = 0; i < numDocs; i++) {
single.setDocument(i);
ArrayList<Long> expectedList = new ArrayList<>();
for (int j = 0; j < single.count(); j++) {
expectedList.add(single.valueAt(j));
}
multi.setDocument(i);
assertEquals(expectedList.size(), multi.count());
for (int j = 0; j < single.count(); j++) {
assertEquals(expectedList.get(j).longValue(), multi.valueAt(j));
}
}
}
ir.close();
ir2.close();
dir.close();
}
@Test
public void testReservedChars() throws Exception {
CharsRefBuilder charsRefBuilder = new CharsRefBuilder();
charsRefBuilder.append("sugg");
charsRefBuilder.setCharAt(2, (char) ContextSuggestField.CONTEXT_SEPARATOR);
Analyzer analyzer = new MockAnalyzer(random());
Document document = new Document();
try (RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "name"))) {
// exception should be thrown for context value containing CONTEXT_SEPARATOR
IllegalArgumentException expected =
expectThrows(
IllegalArgumentException.class,
() -> {
document.add(
new ContextSuggestField("name", "sugg", 1, charsRefBuilder.toString()));
iw.addDocument(document);
iw.commit();
});
assertTrue(expected.getMessage().contains("[0x1d]"));
}
document.clear();
try (RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "name"))) {
// exception should be thrown for context value containing CONTEXT_SEPARATOR
IllegalArgumentException expected =
expectThrows(
IllegalArgumentException.class,
() -> {
document.add(
new ContextSuggestField("name", charsRefBuilder.toString(), 1, "sugg"));
iw.addDocument(document);
iw.commit();
});
assertTrue(expected.getMessage().contains("[0x1d]"));
}
}
// make sure we never reuse from another reader even if it is the same field & codec etc
public void testReuseDocsEnumDifferentReader() throws IOException {
Directory dir = newDirectory();
Codec cp = TestUtil.alwaysPostingsFormat(new Lucene40RWPostingsFormat());
MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
RandomIndexWriter writer =
new RandomIndexWriter(random(), dir, newIndexWriterConfig(analyzer).setCodec(cp));
int numdocs = atLeast(20);
createRandomIndex(numdocs, writer, random());
writer.commit();
DirectoryReader firstReader = DirectoryReader.open(dir);
DirectoryReader secondReader = DirectoryReader.open(dir);
List<LeafReaderContext> leaves = firstReader.leaves();
List<LeafReaderContext> leaves2 = secondReader.leaves();
for (LeafReaderContext ctx : leaves) {
Terms terms = ctx.reader().terms("body");
TermsEnum iterator = terms.iterator();
IdentityHashMap<PostingsEnum, Boolean> enums = new IdentityHashMap<>();
MatchNoBits bits = new Bits.MatchNoBits(firstReader.maxDoc());
iterator = terms.iterator();
PostingsEnum docs = null;
BytesRef term = null;
while ((term = iterator.next()) != null) {
docs =
iterator.postings(
null,
randomDocsEnum("body", term, leaves2, bits),
random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(terms.size(), enums.size());
iterator = terms.iterator();
enums.clear();
docs = null;
while ((term = iterator.next()) != null) {
docs =
iterator.postings(
bits,
randomDocsEnum("body", term, leaves2, bits),
random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(terms.size(), enums.size());
}
writer.close();
IOUtils.close(firstReader, secondReader, dir);
}
@Test
public void testRealisticKeys() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
LineFileDocs lineFileDocs = new LineFileDocs(random());
int num = Math.min(1000, atLeast(100));
Map<String, Integer> mappings = new HashMap<>();
for (int i = 0; i < num; i++) {
Document document = lineFileDocs.nextDoc();
String title = document.getField("title").stringValue();
int weight = Math.abs(random().nextInt());
Integer prevWeight = mappings.get(title);
if (prevWeight == null || prevWeight < weight) {
mappings.put(title, weight);
}
Document doc = new Document();
doc.add(new SuggestField("suggest_field", title, weight));
iw.addDocument(doc);
if (rarely()) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
for (Map.Entry<String, Integer> entry : mappings.entrySet()) {
String title = entry.getKey();
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", title));
TopSuggestDocs suggest = indexSearcher.suggest(query, mappings.size());
assertTrue(suggest.totalHits > 0);
boolean matched = false;
for (ScoreDoc scoreDoc : suggest.scoreDocs) {
matched = Float.compare(scoreDoc.score, (float) entry.getValue()) == 0;
if (matched) {
break;
}
}
assertTrue("at least one of the entries should have the score", matched);
}
reader.close();
iw.close();
}
public void testDocsWithField() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(random(), null);
iwc.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
int numDocs = TEST_NIGHTLY ? atLeast(500) : atLeast(50);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
if (random().nextInt(4) >= 0) {
doc.add(new NumericDocValuesField("numbers", random().nextLong()));
}
doc.add(new NumericDocValuesField("numbersAlways", random().nextLong()));
iw.addDocument(doc);
if (random().nextInt(17) == 0) {
iw.commit();
}
}
DirectoryReader ir = iw.getReader();
iw.forceMerge(1);
DirectoryReader ir2 = iw.getReader();
LeafReader merged = getOnlyLeafReader(ir2);
iw.close();
Bits multi = MultiDocValues.getDocsWithField(ir, "numbers");
Bits single = merged.getDocsWithField("numbers");
if (multi == null) {
assertNull(single);
} else {
assertEquals(single.length(), multi.length());
for (int i = 0; i < numDocs; i++) {
assertEquals(single.get(i), multi.get(i));
}
}
multi = MultiDocValues.getDocsWithField(ir, "numbersAlways");
single = merged.getDocsWithField("numbersAlways");
assertEquals(single.length(), multi.length());
for (int i = 0; i < numDocs; i++) {
assertEquals(single.get(i), multi.get(i));
}
ir.close();
ir2.close();
dir.close();
}
@Test
public void testScoring() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
int num = Math.min(1000, atLeast(100));
String[] prefixes = {"abc", "bac", "cab"};
Map<String, Integer> mappings = new HashMap<>();
for (int i = 0; i < num; i++) {
Document document = new Document();
String suggest =
prefixes[i % 3] + TestUtil.randomSimpleString(random(), 10) + "_" + String.valueOf(i);
int weight = Math.abs(random().nextInt());
document.add(new SuggestField("suggest_field", suggest, weight));
mappings.put(suggest, weight);
iw.addDocument(document);
if (usually()) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
for (String prefix : prefixes) {
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", prefix));
TopSuggestDocs suggest = indexSearcher.suggest(query, num);
assertTrue(suggest.totalHits > 0);
float topScore = -1;
for (SuggestScoreDoc scoreDoc : suggest.scoreLookupDocs()) {
if (topScore != -1) {
assertTrue(topScore >= scoreDoc.score);
}
topScore = scoreDoc.score;
assertThat((float) mappings.get(scoreDoc.key.toString()), equalTo(scoreDoc.score));
assertNotNull(mappings.remove(scoreDoc.key.toString()));
}
}
assertThat(mappings.size(), equalTo(0));
reader.close();
iw.close();
}
public void testSorted() throws Exception {
Directory dir = newDirectory();
Document doc = new Document();
Field field = new SortedDocValuesField("bytes", new BytesRef());
doc.add(field);
IndexWriterConfig iwc = newIndexWriterConfig(random(), null);
iwc.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
int numDocs = TEST_NIGHTLY ? atLeast(500) : atLeast(50);
for (int i = 0; i < numDocs; i++) {
BytesRef ref = new BytesRef(TestUtil.randomUnicodeString(random()));
field.setBytesValue(ref);
if (random().nextInt(7) == 0) {
iw.addDocument(new Document());
}
iw.addDocument(doc);
if (random().nextInt(17) == 0) {
iw.commit();
}
}
DirectoryReader ir = iw.getReader();
iw.forceMerge(1);
DirectoryReader ir2 = iw.getReader();
LeafReader merged = getOnlyLeafReader(ir2);
iw.close();
SortedDocValues multi = MultiDocValues.getSortedValues(ir, "bytes");
SortedDocValues single = merged.getSortedDocValues("bytes");
assertEquals(single.getValueCount(), multi.getValueCount());
for (int i = 0; i < numDocs; i++) {
// check ord
assertEquals(single.getOrd(i), multi.getOrd(i));
// check value
final BytesRef expected = BytesRef.deepCopyOf(single.get(i));
final BytesRef actual = multi.get(i);
assertEquals(expected, actual);
}
ir.close();
ir2.close();
dir.close();
}
@Test
public void testMixedSuggestFields() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
Document document = new Document();
document.add(new SuggestField("suggest_field", "suggestion1", 4));
document.add(new ContextSuggestField("suggest_field", "suggestion2", 3));
try (RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"))) {
// mixing suggest field types for same field name should error out
IllegalArgumentException expected =
expectThrows(
IllegalArgumentException.class,
() -> {
iw.addDocument(document);
iw.commit();
});
assertTrue(expected.getMessage().contains("mixed types"));
}
}
/** Make sure an open IndexWriter on an incoming Directory causes a LockObtainFailedException */
public void testLocksBlock() throws Exception {
Directory src = newDirectory();
RandomIndexWriter w1 = new RandomIndexWriter(random(), src);
w1.addDocument(new Document());
w1.commit();
Directory dest = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setWriteLockTimeout(1);
RandomIndexWriter w2 = new RandomIndexWriter(random(), dest, iwc);
try {
w2.addIndexes(src);
fail("did not hit expected exception");
} catch (LockObtainFailedException lofe) {
// expected
}
IOUtils.close(w1, w2, src, dest);
}
@Test
public void testMultipleSuggestFieldsPerDoc() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(
random(), dir, iwcWithSuggestField(analyzer, "sug_field_1", "sug_field_2"));
Document document = new Document();
document.add(new SuggestField("sug_field_1", "apple", 4));
document.add(new SuggestField("sug_field_2", "april", 3));
iw.addDocument(document);
document = new Document();
document.add(new SuggestField("sug_field_1", "aples", 3));
document.add(new SuggestField("sug_field_2", "apartment", 2));
iw.addDocument(document);
if (rarely()) {
iw.commit();
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("sug_field_1", "ap"));
TopSuggestDocs suggestDocs1 = suggestIndexSearcher.suggest(query, 4);
assertSuggestions(suggestDocs1, new Entry("apple", 4), new Entry("aples", 3));
query = new PrefixCompletionQuery(analyzer, new Term("sug_field_2", "ap"));
TopSuggestDocs suggestDocs2 = suggestIndexSearcher.suggest(query, 4);
assertSuggestions(suggestDocs2, new Entry("april", 3), new Entry("apartment", 2));
// check that the doc ids are consistent
for (int i = 0; i < suggestDocs1.scoreDocs.length; i++) {
ScoreDoc suggestScoreDoc = suggestDocs1.scoreDocs[i];
assertThat(suggestScoreDoc.doc, equalTo(suggestDocs2.scoreDocs[i].doc));
}
reader.close();
iw.close();
}
public void testCacheScoresIfNecessary() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
iw.addDocument(new Document());
iw.commit();
DirectoryReader reader = iw.getReader();
iw.close();
final LeafReaderContext ctx = reader.leaves().get(0);
try {
collector(false, ScoreCachingWrappingScorer.class)
.getLeafCollector(ctx)
.setScorer(new FakeScorer());
fail(
"The collector was configured to expect a ScoreCachingWrappingScorer and did not fail when pass in a FakeScorer");
} catch (AssertionError e) {
// expected
}
// no collector needs scores => no caching
Collector c1 = collector(false, FakeScorer.class);
Collector c2 = collector(false, FakeScorer.class);
MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new FakeScorer());
// only one collector needs scores => no caching
c1 = collector(true, FakeScorer.class);
c2 = collector(false, FakeScorer.class);
MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new FakeScorer());
// several collectors need scores => caching
c1 = collector(true, ScoreCachingWrappingScorer.class);
c2 = collector(true, ScoreCachingWrappingScorer.class);
MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new FakeScorer());
reader.close();
dir.close();
}
@Test
public void testFuzzyFilteredContextQuery() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
Document document = new Document();
document.add(new ContextSuggestField("suggest_field", "sduggestion", 1, "type1"));
document.add(new ContextSuggestField("suggest_field", "sudggestion", 1, "type2"));
document.add(new ContextSuggestField("suggest_field", "sugdgestion", 1, "type3"));
iw.addDocument(document);
document = new Document();
document.add(new ContextSuggestField("suggest_field", "suggdestion", 1, "type4"));
document.add(new ContextSuggestField("suggest_field", "suggestion", 1, "type4"));
iw.addDocument(document);
if (rarely()) {
iw.commit();
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher suggestIndexSearcher = new SuggestIndexSearcher(reader);
CompletionQuery fuzzyQuery =
new FuzzyCompletionQuery(analyzer, new Term("suggest_field", "sugge"));
ContextQuery contextQuery = new ContextQuery(fuzzyQuery);
contextQuery.addContext("type1", 6);
contextQuery.addContext("type3", 2);
TopSuggestDocs suggest = suggestIndexSearcher.suggest(contextQuery, 5);
assertSuggestions(
suggest,
new Entry("sduggestion", "type1", 1 * (1 + 6)),
new Entry("sugdgestion", "type3", 1 * (3 + 2)));
reader.close();
iw.close();
}
@Test
public void testReturnedDocID() throws Exception {
Analyzer analyzer = new MockAnalyzer(random());
RandomIndexWriter iw =
new RandomIndexWriter(random(), dir, iwcWithSuggestField(analyzer, "suggest_field"));
int num = Math.min(1000, atLeast(10));
for (int i = 0; i < num; i++) {
Document document = new Document();
document.add(new SuggestField("suggest_field", "abc_" + i, num));
document.add(new StoredField("int_field", i));
iw.addDocument(document);
if (random().nextBoolean()) {
iw.commit();
}
}
DirectoryReader reader = iw.getReader();
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
PrefixCompletionQuery query =
new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
TopSuggestDocs suggest = indexSearcher.suggest(query, num);
assertEquals(num, suggest.totalHits);
for (SuggestScoreDoc suggestScoreDoc : suggest.scoreLookupDocs()) {
String key = suggestScoreDoc.key.toString();
assertTrue(key.startsWith("abc_"));
String substring = key.substring(4);
int fieldValue = Integer.parseInt(substring);
StoredDocument doc = reader.document(suggestScoreDoc.doc);
assertEquals(doc.getField("int_field").numericValue().intValue(), fieldValue);
}
reader.close();
iw.close();
}