private void doTest(Random random, PrintWriter out, boolean useCompoundFiles, int MAX_DOCS)
throws Exception {
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random);
IndexWriterConfig conf = newIndexWriterConfig(analyzer);
final MergePolicy mp = conf.getMergePolicy();
mp.setNoCFSRatio(useCompoundFiles ? 1.0 : 0.0);
IndexWriter writer = new IndexWriter(directory, conf);
if (VERBOSE) {
System.out.println("TEST: now build index MAX_DOCS=" + MAX_DOCS);
}
for (int j = 0; j < MAX_DOCS; j++) {
Document d = new Document();
d.add(newTextField(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES));
d.add(newTextField(ID_FIELD, Integer.toString(j), Field.Store.YES));
writer.addDocument(d);
}
writer.close();
// try a search without OR
IndexReader reader = DirectoryReader.open(directory);
IndexSearcher searcher = newSearcher(reader);
Query query = new TermQuery(new Term(PRIORITY_FIELD, HIGH_PRIORITY));
out.println("Query: " + query.toString(PRIORITY_FIELD));
if (VERBOSE) {
System.out.println("TEST: search query=" + query);
}
final Sort sort = new Sort(SortField.FIELD_SCORE, new SortField(ID_FIELD, SortField.Type.INT));
ScoreDoc[] hits = searcher.search(query, null, MAX_DOCS, sort).scoreDocs;
printHits(out, hits, searcher);
checkHits(hits, MAX_DOCS, searcher);
// try a new search with OR
searcher = newSearcher(reader);
hits = null;
BooleanQuery booleanQuery = new BooleanQuery();
booleanQuery.add(
new TermQuery(new Term(PRIORITY_FIELD, HIGH_PRIORITY)), BooleanClause.Occur.SHOULD);
booleanQuery.add(
new TermQuery(new Term(PRIORITY_FIELD, MED_PRIORITY)), BooleanClause.Occur.SHOULD);
out.println("Query: " + booleanQuery.toString(PRIORITY_FIELD));
hits = searcher.search(booleanQuery, null, MAX_DOCS, sort).scoreDocs;
printHits(out, hits, searcher);
checkHits(hits, MAX_DOCS, searcher);
reader.close();
directory.close();
}
// LUCENE-1274: test writer.prepareCommit()
public void testPrepareCommitRollback() throws IOException {
MockDirectoryWrapper dir = newDirectory();
dir.setPreventDoubleWrite(false);
IndexWriter writer =
new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMaxBufferedDocs(2)
.setMergePolicy(newLogMergePolicy(5)));
writer.commit();
for (int i = 0; i < 23; i++) TestIndexWriter.addDoc(writer);
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(0, reader.numDocs());
writer.prepareCommit();
IndexReader reader2 = DirectoryReader.open(dir);
assertEquals(0, reader2.numDocs());
writer.rollback();
IndexReader reader3 = DirectoryReader.openIfChanged(reader);
assertNull(reader3);
assertEquals(0, reader.numDocs());
assertEquals(0, reader2.numDocs());
reader.close();
reader2.close();
writer =
new IndexWriter(
dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
for (int i = 0; i < 17; i++) TestIndexWriter.addDoc(writer);
reader = DirectoryReader.open(dir);
assertEquals(0, reader.numDocs());
reader.close();
writer.prepareCommit();
reader = DirectoryReader.open(dir);
assertEquals(0, reader.numDocs());
reader.close();
writer.commit();
reader = DirectoryReader.open(dir);
assertEquals(17, reader.numDocs());
reader.close();
writer.close();
dir.close();
}
// LUCENE-1274
public void testPrepareCommitNoChanges() throws IOException {
Directory dir = newDirectory();
IndexWriter writer =
new IndexWriter(
dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
writer.prepareCommit();
writer.commit();
writer.close();
IndexReader reader = DirectoryReader.open(dir);
assertEquals(0, reader.numDocs());
reader.close();
dir.close();
}
/*
* Simple test for "commit on close": open writer then
* add a bunch of docs, making sure reader does not see
* these docs until writer is closed.
*/
public void testCommitOnClose() throws IOException {
Directory dir = newDirectory();
IndexWriter writer =
new IndexWriter(
dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
for (int i = 0; i < 14; i++) {
TestIndexWriter.addDoc(writer);
}
writer.close();
Term searchTerm = new Term("content", "aaa");
DirectoryReader reader = DirectoryReader.open(dir);
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("first number of hits", 14, hits.length);
reader.close();
reader = DirectoryReader.open(dir);
writer =
new IndexWriter(
dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 11; j++) {
TestIndexWriter.addDoc(writer);
}
IndexReader r = DirectoryReader.open(dir);
searcher = new IndexSearcher(r);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
r.close();
assertTrue("reader should have still been current", reader.isCurrent());
}
// Now, close the writer:
writer.close();
assertFalse("reader should not be current now", reader.isCurrent());
IndexReader r = DirectoryReader.open(dir);
searcher = new IndexSearcher(r);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("reader did not see changes after writer was closed", 47, hits.length);
r.close();
reader.close();
dir.close();
}
public void testTermDocsEnum() throws Exception {
Directory dir = newDirectory();
IndexWriter w =
new IndexWriter(
dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document d = new Document();
d.add(newStringField("f", "j", Field.Store.NO));
w.addDocument(d);
w.commit();
w.addDocument(d);
IndexReader r = w.getReader();
w.close();
DocsEnum de = MultiFields.getTermDocsEnum(r, null, "f", new BytesRef("j"));
assertEquals(0, de.nextDoc());
assertEquals(1, de.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, de.nextDoc());
r.close();
dir.close();
}
public void testSeparateEnums() throws Exception {
Directory dir = newDirectory();
IndexWriter w =
new IndexWriter(
dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document d = new Document();
d.add(newStringField("f", "j", Field.Store.NO));
w.addDocument(d);
w.commit();
w.addDocument(d);
IndexReader r = w.getReader();
w.close();
DocsEnum d1 = _TestUtil.docs(random(), r, "f", new BytesRef("j"), null, null, 0);
DocsEnum d2 = _TestUtil.docs(random(), r, "f", new BytesRef("j"), null, null, 0);
assertEquals(0, d1.nextDoc());
assertEquals(0, d2.nextDoc());
r.close();
dir.close();
}
public void testRandom() throws Exception {
int num = atLeast(2);
for (int iter = 0; iter < num; iter++) {
if (VERBOSE) {
System.out.println("TEST: iter=" + iter);
}
Directory dir = newDirectory();
IndexWriter w =
new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(NoMergePolicy.COMPOUND_FILES));
_TestUtil.keepFullyDeletedSegments(w);
Map<BytesRef, List<Integer>> docs = new HashMap<BytesRef, List<Integer>>();
Set<Integer> deleted = new HashSet<Integer>();
List<BytesRef> terms = new ArrayList<BytesRef>();
int numDocs = _TestUtil.nextInt(random(), 1, 100 * RANDOM_MULTIPLIER);
Document doc = new Document();
Field f = newStringField("field", "", Field.Store.NO);
doc.add(f);
Field id = newStringField("id", "", Field.Store.NO);
doc.add(id);
boolean onlyUniqueTerms = random().nextBoolean();
if (VERBOSE) {
System.out.println("TEST: onlyUniqueTerms=" + onlyUniqueTerms + " numDocs=" + numDocs);
}
Set<BytesRef> uniqueTerms = new HashSet<BytesRef>();
for (int i = 0; i < numDocs; i++) {
if (!onlyUniqueTerms && random().nextBoolean() && terms.size() > 0) {
// re-use existing term
BytesRef term = terms.get(random().nextInt(terms.size()));
docs.get(term).add(i);
f.setStringValue(term.utf8ToString());
} else {
String s = _TestUtil.randomUnicodeString(random(), 10);
BytesRef term = new BytesRef(s);
if (!docs.containsKey(term)) {
docs.put(term, new ArrayList<Integer>());
}
docs.get(term).add(i);
terms.add(term);
uniqueTerms.add(term);
f.setStringValue(s);
}
id.setStringValue("" + i);
w.addDocument(doc);
if (random().nextInt(4) == 1) {
w.commit();
}
if (i > 0 && random().nextInt(20) == 1) {
int delID = random().nextInt(i);
deleted.add(delID);
w.deleteDocuments(new Term("id", "" + delID));
if (VERBOSE) {
System.out.println("TEST: delete " + delID);
}
}
}
if (VERBOSE) {
List<BytesRef> termsList = new ArrayList<BytesRef>(uniqueTerms);
Collections.sort(termsList, BytesRef.getUTF8SortedAsUTF16Comparator());
System.out.println("TEST: terms in UTF16 order:");
for (BytesRef b : termsList) {
System.out.println(" " + UnicodeUtil.toHexString(b.utf8ToString()) + " " + b);
for (int docID : docs.get(b)) {
if (deleted.contains(docID)) {
System.out.println(" " + docID + " (deleted)");
} else {
System.out.println(" " + docID);
}
}
}
}
IndexReader reader = w.getReader();
w.close();
if (VERBOSE) {
System.out.println("TEST: reader=" + reader);
}
Bits liveDocs = MultiFields.getLiveDocs(reader);
for (int delDoc : deleted) {
assertFalse(liveDocs.get(delDoc));
}
for (int i = 0; i < 100; i++) {
BytesRef term = terms.get(random().nextInt(terms.size()));
if (VERBOSE) {
System.out.println(
"TEST: seek term=" + UnicodeUtil.toHexString(term.utf8ToString()) + " " + term);
}
DocsEnum docsEnum = _TestUtil.docs(random(), reader, "field", term, liveDocs, null, 0);
assertNotNull(docsEnum);
for (int docID : docs.get(term)) {
if (!deleted.contains(docID)) {
assertEquals(docID, docsEnum.nextDoc());
}
}
assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsEnum.nextDoc());
}
reader.close();
dir.close();
}
}
/*
* Simple test for "commit on close": open writer, then
* add a bunch of docs, making sure reader does not see
* them until writer has closed. Then instead of
* closing the writer, call abort and verify reader sees
* nothing was added. Then verify we can open the index
* and add docs to it.
*/
public void testCommitOnCloseAbort() throws IOException {
MockDirectoryWrapper dir = newDirectory();
IndexWriter writer =
new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMaxBufferedDocs(10));
for (int i = 0; i < 14; i++) {
TestIndexWriter.addDoc(writer);
}
writer.close();
Term searchTerm = new Term("content", "aaa");
IndexReader reader = DirectoryReader.open(dir);
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("first number of hits", 14, hits.length);
reader.close();
writer =
new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setOpenMode(OpenMode.APPEND)
.setMaxBufferedDocs(10));
for (int j = 0; j < 17; j++) {
TestIndexWriter.addDoc(writer);
}
// Delete all docs:
writer.deleteDocuments(searchTerm);
reader = DirectoryReader.open(dir);
searcher = new IndexSearcher(reader);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
reader.close();
// Now, close the writer:
writer.rollback();
TestIndexWriter.assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()");
reader = DirectoryReader.open(dir);
searcher = new IndexSearcher(reader);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("saw changes after writer.abort", 14, hits.length);
reader.close();
// Now make sure we can re-open the index, add docs,
// and all is good:
writer =
new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setOpenMode(OpenMode.APPEND)
.setMaxBufferedDocs(10));
// On abort, writer in fact may write to the same
// segments_N file:
dir.setPreventDoubleWrite(false);
for (int i = 0; i < 12; i++) {
for (int j = 0; j < 17; j++) {
TestIndexWriter.addDoc(writer);
}
IndexReader r = DirectoryReader.open(dir);
searcher = new IndexSearcher(r);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
r.close();
}
writer.close();
IndexReader r = DirectoryReader.open(dir);
searcher = new IndexSearcher(r);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("didn't see changes after close", 218, hits.length);
r.close();
dir.close();
}