// test when delete terms only apply to disk segments public void testNonRAMDelete() throws IOException { Directory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setMaxBufferedDocs(2); modifier.setMaxBufferedDeleteTerms(2); int id = 0; int value = 100; for (int i = 0; i < 7; i++) { addDoc(modifier, ++id, value); } modifier.commit(); assertEquals(0, modifier.getNumBufferedDocuments()); assertTrue(0 < modifier.getSegmentCount()); modifier.commit(); IndexReader reader = IndexReader.open(dir, true); assertEquals(7, reader.numDocs()); reader.close(); modifier.deleteDocuments(new Term("value", String.valueOf(value))); modifier.commit(); reader = IndexReader.open(dir, true); assertEquals(0, reader.numDocs()); reader.close(); modifier.close(); dir.close(); }
private void setUpDirs(Directory dir, Directory aux, boolean withID) throws IOException { IndexWriter writer = null; writer = newWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setOpenMode(OpenMode.CREATE) .setMaxBufferedDocs(1000)); // add 1000 documents in 1 segment if (withID) { addDocsWithID(writer, 1000, 0); } else { addDocs(writer, 1000); } assertEquals(1000, writer.maxDoc()); assertEquals(1, writer.getSegmentCount()); writer.close(); writer = newWriter( aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setOpenMode(OpenMode.CREATE) .setMaxBufferedDocs(1000) .setMergePolicy(newLogMergePolicy(false, 10))); // add 30 documents in 3 segments for (int i = 0; i < 3; i++) { if (withID) { addDocsWithID(writer, 10, 10 * i); } else { addDocs(writer, 10); } writer.close(); writer = newWriter( aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setOpenMode(OpenMode.APPEND) .setMaxBufferedDocs(1000) .setMergePolicy(newLogMergePolicy(false, 10))); } assertEquals(30, writer.maxDoc()); assertEquals(3, writer.getSegmentCount()); writer.close(); }
public void testForceMergeNotNeeded() throws IOException { try (Directory dir = newDirectory()) { final AtomicBoolean mayMerge = new AtomicBoolean(true); final MergeScheduler mergeScheduler = new SerialMergeScheduler() { @Override public synchronized void merge( IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws IOException { if (mayMerge.get() == false) { MergePolicy.OneMerge merge = writer.getNextMerge(); if (merge != null) { System.out.println( "TEST: we should not need any merging, yet merge policy returned merge " + merge); throw new AssertionError(); } } super.merge(writer, trigger, newMergesFound); } }; MergePolicy mp = mergePolicy(); assumeFalse( "this test cannot tolerate random forceMerges", mp.toString().contains("MockRandomMergePolicy")); mp.setNoCFSRatio(random().nextBoolean() ? 0 : 1); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setMergeScheduler(mergeScheduler); iwc.setMergePolicy(mp); IndexWriter writer = new IndexWriter(dir, iwc); final int numSegments = TestUtil.nextInt(random(), 2, 20); for (int i = 0; i < numSegments; ++i) { final int numDocs = TestUtil.nextInt(random(), 1, 5); for (int j = 0; j < numDocs; ++j) { writer.addDocument(new Document()); } writer.getReader().close(); } for (int i = 5; i >= 0; --i) { final int segmentCount = writer.getSegmentCount(); final int maxNumSegments = i == 0 ? 1 : TestUtil.nextInt(random(), 1, 10); mayMerge.set(segmentCount > maxNumSegments); if (VERBOSE) { System.out.println( "TEST: now forceMerge(maxNumSegments=" + maxNumSegments + ") vs segmentCount=" + segmentCount); } writer.forceMerge(maxNumSegments); } writer.close(); } }
// test when delete terms only apply to ram segments public void testRAMDeletes() throws IOException { for (int t = 0; t < 2; t++) { Directory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter( dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setMaxBufferedDocs(4); modifier.setMaxBufferedDeleteTerms(4); int id = 0; int value = 100; addDoc(modifier, ++id, value); if (0 == t) modifier.deleteDocuments(new Term("value", String.valueOf(value))); else modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value)))); addDoc(modifier, ++id, value); if (0 == t) { modifier.deleteDocuments(new Term("value", String.valueOf(value))); assertEquals(2, modifier.getNumBufferedDeleteTerms()); assertEquals(1, modifier.getBufferedDeleteTermsSize()); } else modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value)))); addDoc(modifier, ++id, value); assertEquals(0, modifier.getSegmentCount()); modifier.commit(); modifier.commit(); IndexReader reader = IndexReader.open(dir, true); assertEquals(1, reader.numDocs()); int hitCount = getHitCount(dir, new Term("id", String.valueOf(id))); assertEquals(1, hitCount); reader.close(); modifier.close(); dir.close(); } }
// case 5: tail segments, invariants not hold public void testMoreMerges() throws IOException { // main directory Directory dir = newDirectory(); // auxiliary directory Directory aux = newDirectory(); Directory aux2 = newDirectory(); setUpDirs(dir, aux, true); IndexWriter writer = newWriter( aux2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setOpenMode(OpenMode.CREATE) .setMaxBufferedDocs(100) .setMergePolicy(newLogMergePolicy(10))); writer.addIndexes(aux); assertEquals(30, writer.maxDoc()); assertEquals(3, writer.getSegmentCount()); writer.close(); IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setMergePolicy(NoMergePolicy.COMPOUND_FILES); writer = new IndexWriter(aux, dontMergeConfig); for (int i = 0; i < 27; i++) { writer.deleteDocuments(new Term("id", "" + i)); } writer.close(); IndexReader reader = DirectoryReader.open(aux); assertEquals(3, reader.numDocs()); reader.close(); dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setMergePolicy(NoMergePolicy.COMPOUND_FILES); writer = new IndexWriter(aux2, dontMergeConfig); for (int i = 0; i < 8; i++) { writer.deleteDocuments(new Term("id", "" + i)); } writer.close(); reader = DirectoryReader.open(aux2); assertEquals(22, reader.numDocs()); reader.close(); writer = newWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setOpenMode(OpenMode.APPEND) .setMaxBufferedDocs(6) .setMergePolicy(newLogMergePolicy(4))); writer.addIndexes(aux, aux2); assertEquals(1040, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); dir.close(); aux.close(); aux2.close(); }