// test when delete terms only apply to disk segments
public void testNonRAMDelete() throws IOException {
Directory dir = new MockRAMDirectory();
IndexWriter modifier =
new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
modifier.setMaxBufferedDocs(2);
modifier.setMaxBufferedDeleteTerms(2);
int id = 0;
int value = 100;
for (int i = 0; i < 7; i++) {
addDoc(modifier, ++id, value);
}
modifier.commit();
assertEquals(0, modifier.getNumBufferedDocuments());
assertTrue(0 < modifier.getSegmentCount());
modifier.commit();
IndexReader reader = IndexReader.open(dir, true);
assertEquals(7, reader.numDocs());
reader.close();
modifier.deleteDocuments(new Term("value", String.valueOf(value)));
modifier.commit();
reader = IndexReader.open(dir, true);
assertEquals(0, reader.numDocs());
reader.close();
modifier.close();
dir.close();
}
private void setUpDirs(Directory dir, Directory aux, boolean withID) throws IOException {
IndexWriter writer = null;
writer =
newWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setOpenMode(OpenMode.CREATE)
.setMaxBufferedDocs(1000));
// add 1000 documents in 1 segment
if (withID) {
addDocsWithID(writer, 1000, 0);
} else {
addDocs(writer, 1000);
}
assertEquals(1000, writer.maxDoc());
assertEquals(1, writer.getSegmentCount());
writer.close();
writer =
newWriter(
aux,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setOpenMode(OpenMode.CREATE)
.setMaxBufferedDocs(1000)
.setMergePolicy(newLogMergePolicy(false, 10)));
// add 30 documents in 3 segments
for (int i = 0; i < 3; i++) {
if (withID) {
addDocsWithID(writer, 10, 10 * i);
} else {
addDocs(writer, 10);
}
writer.close();
writer =
newWriter(
aux,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setOpenMode(OpenMode.APPEND)
.setMaxBufferedDocs(1000)
.setMergePolicy(newLogMergePolicy(false, 10)));
}
assertEquals(30, writer.maxDoc());
assertEquals(3, writer.getSegmentCount());
writer.close();
}
public void testForceMergeNotNeeded() throws IOException {
try (Directory dir = newDirectory()) {
final AtomicBoolean mayMerge = new AtomicBoolean(true);
final MergeScheduler mergeScheduler =
new SerialMergeScheduler() {
@Override
public synchronized void merge(
IndexWriter writer, MergeTrigger trigger, boolean newMergesFound)
throws IOException {
if (mayMerge.get() == false) {
MergePolicy.OneMerge merge = writer.getNextMerge();
if (merge != null) {
System.out.println(
"TEST: we should not need any merging, yet merge policy returned merge "
+ merge);
throw new AssertionError();
}
}
super.merge(writer, trigger, newMergesFound);
}
};
MergePolicy mp = mergePolicy();
assumeFalse(
"this test cannot tolerate random forceMerges",
mp.toString().contains("MockRandomMergePolicy"));
mp.setNoCFSRatio(random().nextBoolean() ? 0 : 1);
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
iwc.setMergeScheduler(mergeScheduler);
iwc.setMergePolicy(mp);
IndexWriter writer = new IndexWriter(dir, iwc);
final int numSegments = TestUtil.nextInt(random(), 2, 20);
for (int i = 0; i < numSegments; ++i) {
final int numDocs = TestUtil.nextInt(random(), 1, 5);
for (int j = 0; j < numDocs; ++j) {
writer.addDocument(new Document());
}
writer.getReader().close();
}
for (int i = 5; i >= 0; --i) {
final int segmentCount = writer.getSegmentCount();
final int maxNumSegments = i == 0 ? 1 : TestUtil.nextInt(random(), 1, 10);
mayMerge.set(segmentCount > maxNumSegments);
if (VERBOSE) {
System.out.println(
"TEST: now forceMerge(maxNumSegments="
+ maxNumSegments
+ ") vs segmentCount="
+ segmentCount);
}
writer.forceMerge(maxNumSegments);
}
writer.close();
}
}
// test when delete terms only apply to ram segments
public void testRAMDeletes() throws IOException {
for (int t = 0; t < 2; t++) {
Directory dir = new MockRAMDirectory();
IndexWriter modifier =
new IndexWriter(
dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
modifier.setMaxBufferedDocs(4);
modifier.setMaxBufferedDeleteTerms(4);
int id = 0;
int value = 100;
addDoc(modifier, ++id, value);
if (0 == t) modifier.deleteDocuments(new Term("value", String.valueOf(value)));
else modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value))));
addDoc(modifier, ++id, value);
if (0 == t) {
modifier.deleteDocuments(new Term("value", String.valueOf(value)));
assertEquals(2, modifier.getNumBufferedDeleteTerms());
assertEquals(1, modifier.getBufferedDeleteTermsSize());
} else modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value))));
addDoc(modifier, ++id, value);
assertEquals(0, modifier.getSegmentCount());
modifier.commit();
modifier.commit();
IndexReader reader = IndexReader.open(dir, true);
assertEquals(1, reader.numDocs());
int hitCount = getHitCount(dir, new Term("id", String.valueOf(id)));
assertEquals(1, hitCount);
reader.close();
modifier.close();
dir.close();
}
}
// case 5: tail segments, invariants not hold
public void testMoreMerges() throws IOException {
// main directory
Directory dir = newDirectory();
// auxiliary directory
Directory aux = newDirectory();
Directory aux2 = newDirectory();
setUpDirs(dir, aux, true);
IndexWriter writer =
newWriter(
aux2,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setOpenMode(OpenMode.CREATE)
.setMaxBufferedDocs(100)
.setMergePolicy(newLogMergePolicy(10)));
writer.addIndexes(aux);
assertEquals(30, writer.maxDoc());
assertEquals(3, writer.getSegmentCount());
writer.close();
IndexWriterConfig dontMergeConfig =
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(NoMergePolicy.COMPOUND_FILES);
writer = new IndexWriter(aux, dontMergeConfig);
for (int i = 0; i < 27; i++) {
writer.deleteDocuments(new Term("id", "" + i));
}
writer.close();
IndexReader reader = DirectoryReader.open(aux);
assertEquals(3, reader.numDocs());
reader.close();
dontMergeConfig =
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(NoMergePolicy.COMPOUND_FILES);
writer = new IndexWriter(aux2, dontMergeConfig);
for (int i = 0; i < 8; i++) {
writer.deleteDocuments(new Term("id", "" + i));
}
writer.close();
reader = DirectoryReader.open(aux2);
assertEquals(22, reader.numDocs());
reader.close();
writer =
newWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setOpenMode(OpenMode.APPEND)
.setMaxBufferedDocs(6)
.setMergePolicy(newLogMergePolicy(4)));
writer.addIndexes(aux, aux2);
assertEquals(1040, writer.maxDoc());
assertEquals(1000, writer.getDocCount(0));
writer.close();
dir.close();
aux.close();
aux2.close();
}
