// test the simple case public void testSimpleCase() throws IOException { String[] keywords = {"1", "2"}; String[] unindexed = {"Netherlands", "Italy"}; String[] unstored = {"Amsterdam has lots of bridges", "Venice has lots of canals"}; String[] text = {"Amsterdam", "Venice"}; Directory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setUseCompoundFile(true); modifier.setMaxBufferedDeleteTerms(1); for (int i = 0; i < keywords.length; i++) { Document doc = new Document(); doc.add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO)); doc.add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED)); doc.add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED)); modifier.addDocument(doc); } modifier.optimize(); modifier.commit(); Term term = new Term("city", "Amsterdam"); int hitCount = getHitCount(dir, term); assertEquals(1, hitCount); modifier.deleteDocuments(term); modifier.commit(); hitCount = getHitCount(dir, term); assertEquals(0, hitCount); modifier.close(); dir.close(); }
// test when delete terms only apply to disk segments public void testNonRAMDelete() throws IOException { Directory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setMaxBufferedDocs(2); modifier.setMaxBufferedDeleteTerms(2); int id = 0; int value = 100; for (int i = 0; i < 7; i++) { addDoc(modifier, ++id, value); } modifier.commit(); assertEquals(0, modifier.getNumBufferedDocuments()); assertTrue(0 < modifier.getSegmentCount()); modifier.commit(); IndexReader reader = IndexReader.open(dir, true); assertEquals(7, reader.numDocs()); reader.close(); modifier.deleteDocuments(new Term("value", String.valueOf(value))); modifier.commit(); reader = IndexReader.open(dir, true); assertEquals(0, reader.numDocs()); reader.close(); modifier.close(); dir.close(); }
// test rollback of deleteAll() public void testDeleteAllRollback() throws IOException { Directory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setMaxBufferedDocs(2); modifier.setMaxBufferedDeleteTerms(2); int id = 0; int value = 100; for (int i = 0; i < 7; i++) { addDoc(modifier, ++id, value); } modifier.commit(); addDoc(modifier, ++id, value); IndexReader reader = IndexReader.open(dir, true); assertEquals(7, reader.numDocs()); reader.close(); // Delete all modifier.deleteAll(); // Roll it back modifier.rollback(); modifier.close(); // Validate that the docs are still there reader = IndexReader.open(dir, true); assertEquals(7, reader.numDocs()); reader.close(); dir.close(); }
public void testMaxBufferedDeletes() throws IOException { Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); writer.setMaxBufferedDeleteTerms(1); writer.deleteDocuments(new Term("foobar", "1")); writer.deleteDocuments(new Term("foobar", "1")); writer.deleteDocuments(new Term("foobar", "1")); assertEquals(3, writer.getFlushDeletesCount()); writer.close(); dir.close(); }
// test deleteAll() public void testDeleteAll() throws IOException { Directory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setMaxBufferedDocs(2); modifier.setMaxBufferedDeleteTerms(2); int id = 0; int value = 100; for (int i = 0; i < 7; i++) { addDoc(modifier, ++id, value); } modifier.commit(); IndexReader reader = IndexReader.open(dir, true); assertEquals(7, reader.numDocs()); reader.close(); // Add 1 doc (so we will have something buffered) addDoc(modifier, 99, value); // Delete all modifier.deleteAll(); // Delete all shouldn't be on disk yet reader = IndexReader.open(dir, true); assertEquals(7, reader.numDocs()); reader.close(); // Add a doc and update a doc (after the deleteAll, before the commit) addDoc(modifier, 101, value); updateDoc(modifier, 102, value); // commit the delete all modifier.commit(); // Validate there are no docs left reader = IndexReader.open(dir, true); assertEquals(2, reader.numDocs()); reader.close(); modifier.close(); dir.close(); }
// test that batched delete terms are flushed together public void testBatchDeletes() throws IOException { Directory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setMaxBufferedDocs(2); modifier.setMaxBufferedDeleteTerms(2); int id = 0; int value = 100; for (int i = 0; i < 7; i++) { addDoc(modifier, ++id, value); } modifier.commit(); IndexReader reader = IndexReader.open(dir, true); assertEquals(7, reader.numDocs()); reader.close(); id = 0; modifier.deleteDocuments(new Term("id", String.valueOf(++id))); modifier.deleteDocuments(new Term("id", String.valueOf(++id))); modifier.commit(); reader = IndexReader.open(dir, true); assertEquals(5, reader.numDocs()); reader.close(); Term[] terms = new Term[3]; for (int i = 0; i < terms.length; i++) { terms[i] = new Term("id", String.valueOf(++id)); } modifier.deleteDocuments(terms); modifier.commit(); reader = IndexReader.open(dir, true); assertEquals(2, reader.numDocs()); reader.close(); modifier.close(); dir.close(); }
// test when delete terms only apply to ram segments public void testRAMDeletes() throws IOException { for (int t = 0; t < 2; t++) { Directory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter( dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setMaxBufferedDocs(4); modifier.setMaxBufferedDeleteTerms(4); int id = 0; int value = 100; addDoc(modifier, ++id, value); if (0 == t) modifier.deleteDocuments(new Term("value", String.valueOf(value))); else modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value)))); addDoc(modifier, ++id, value); if (0 == t) { modifier.deleteDocuments(new Term("value", String.valueOf(value))); assertEquals(2, modifier.getNumBufferedDeleteTerms()); assertEquals(1, modifier.getBufferedDeleteTermsSize()); } else modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value)))); addDoc(modifier, ++id, value); assertEquals(0, modifier.getSegmentCount()); modifier.commit(); modifier.commit(); IndexReader reader = IndexReader.open(dir, true); assertEquals(1, reader.numDocs()); int hitCount = getHitCount(dir, new Term("id", String.valueOf(id))); assertEquals(1, hitCount); reader.close(); modifier.close(); dir.close(); } }
// This test tests that buffered deletes are cleared when // an Exception is hit during flush. public void testErrorAfterApplyDeletes() throws IOException { MockRAMDirectory.Failure failure = new MockRAMDirectory.Failure() { boolean sawMaybe = false; boolean failed = false; @Override public MockRAMDirectory.Failure reset() { sawMaybe = false; failed = false; return this; } @Override public void eval(MockRAMDirectory dir) throws IOException { if (sawMaybe && !failed) { boolean seen = false; StackTraceElement[] trace = new Exception().getStackTrace(); for (int i = 0; i < trace.length; i++) { if ("applyDeletes".equals(trace[i].getMethodName())) { seen = true; break; } } if (!seen) { // Only fail once we are no longer in applyDeletes failed = true; throw new IOException("fail after applyDeletes"); } } if (!failed) { StackTraceElement[] trace = new Exception().getStackTrace(); for (int i = 0; i < trace.length; i++) { if ("applyDeletes".equals(trace[i].getMethodName())) { sawMaybe = true; break; } } } } }; // create a couple of files String[] keywords = {"1", "2"}; String[] unindexed = {"Netherlands", "Italy"}; String[] unstored = {"Amsterdam has lots of bridges", "Venice has lots of canals"}; String[] text = {"Amsterdam", "Venice"}; MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); modifier.setUseCompoundFile(true); modifier.setMaxBufferedDeleteTerms(2); dir.failOn(failure.reset()); for (int i = 0; i < keywords.length; i++) { Document doc = new Document(); doc.add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO)); doc.add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED)); doc.add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED)); modifier.addDocument(doc); } // flush (and commit if ac) modifier.optimize(); modifier.commit(); // one of the two files hits Term term = new Term("city", "Amsterdam"); int hitCount = getHitCount(dir, term); assertEquals(1, hitCount); // open the writer again (closed above) // delete the doc // max buf del terms is two, so this is buffered modifier.deleteDocuments(term); // add a doc (needed for the !ac case; see below) // doc remains buffered Document doc = new Document(); modifier.addDocument(doc); // commit the changes, the buffered deletes, and the new doc // The failure object will fail on the first write after the del // file gets created when processing the buffered delete // in the ac case, this will be when writing the new segments // files so we really don't need the new doc, but it's harmless // in the !ac case, a new segments file won't be created but in // this case, creation of the cfs file happens next so we need // the doc (to test that it's okay that we don't lose deletes if // failing while creating the cfs file) boolean failed = false; try { modifier.commit(); } catch (IOException ioe) { failed = true; } assertTrue(failed); // The commit above failed, so we need to retry it (which will // succeed, because the failure is a one-shot) modifier.commit(); hitCount = getHitCount(dir, term); // Make sure the delete was successfully flushed: assertEquals(0, hitCount); modifier.close(); dir.close(); }
/** * Make sure if modifier tries to commit but hits disk full that modifier remains consistent and * usable. Similar to TestIndexReader.testDiskFull(). */ private void testOperationsOnDiskFull(boolean updates) throws IOException { boolean debug = false; Term searchTerm = new Term("content", "aaa"); int START_COUNT = 157; int END_COUNT = 144; // First build up a starting index: MockRAMDirectory startDir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter( startDir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); for (int i = 0; i < 157; i++) { Document d = new Document(); d.add(new Field("id", Integer.toString(i), Field.Store.YES, Field.Index.NOT_ANALYZED)); d.add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED)); writer.addDocument(d); } writer.close(); long diskUsage = startDir.sizeInBytes(); long diskFree = diskUsage + 10; IOException err = null; boolean done = false; // Iterate w/ ever increasing free disk space: while (!done) { MockRAMDirectory dir = new MockRAMDirectory(startDir); dir.setPreventDoubleWrite(false); IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); modifier.setMaxBufferedDocs(1000); // use flush or close modifier.setMaxBufferedDeleteTerms(1000); // use flush or close // For each disk size, first try to commit against // dir that will hit random IOExceptions & disk // full; after, give it infinite disk space & turn // off random IOExceptions & retry w/ same reader: boolean success = false; for (int x = 0; x < 2; x++) { double rate = 0.1; double diskRatio = ((double) diskFree) / diskUsage; long thisDiskFree; String testName; if (0 == x) { thisDiskFree = diskFree; if (diskRatio >= 2.0) { rate /= 2; } if (diskRatio >= 4.0) { rate /= 2; } if (diskRatio >= 6.0) { rate = 0.0; } if (debug) { System.out.println("\ncycle: " + diskFree + " bytes"); } testName = "disk full during reader.close() @ " + thisDiskFree + " bytes"; } else { thisDiskFree = 0; rate = 0.0; if (debug) { System.out.println("\ncycle: same writer: unlimited disk space"); } testName = "reader re-use after disk full"; } dir.setMaxSizeInBytes(thisDiskFree); dir.setRandomIOExceptionRate(rate, diskFree); try { if (0 == x) { int docId = 12; for (int i = 0; i < 13; i++) { if (updates) { Document d = new Document(); d.add( new Field( "id", Integer.toString(i), Field.Store.YES, Field.Index.NOT_ANALYZED)); d.add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED)); modifier.updateDocument(new Term("id", Integer.toString(docId)), d); } else { // deletes modifier.deleteDocuments(new Term("id", Integer.toString(docId))); // modifier.setNorm(docId, "contents", (float)2.0); } docId += 12; } } modifier.close(); success = true; if (0 == x) { done = true; } } catch (IOException e) { if (debug) { System.out.println(" hit IOException: " + e); e.printStackTrace(System.out); } err = e; if (1 == x) { e.printStackTrace(); fail(testName + " hit IOException after disk space was freed up"); } } // If the close() succeeded, make sure there are // no unreferenced files. if (success) { _TestUtil.checkIndex(dir); TestIndexWriter.assertNoUnreferencedFiles(dir, "after writer.close"); } // Finally, verify index is not corrupt, and, if // we succeeded, we see all docs changed, and if // we failed, we see either all docs or no docs // changed (transactional semantics): IndexReader newReader = null; try { newReader = IndexReader.open(dir, true); } catch (IOException e) { e.printStackTrace(); fail( testName + ":exception when creating IndexReader after disk full during close: " + e); } IndexSearcher searcher = new IndexSearcher(newReader); ScoreDoc[] hits = null; try { hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; } catch (IOException e) { e.printStackTrace(); fail(testName + ": exception when searching: " + e); } int result2 = hits.length; if (success) { if (x == 0 && result2 != END_COUNT) { fail( testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT); } else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT) { // It's possible that the first exception was // "recoverable" wrt pending deletes, in which // case the pending deletes are retained and // then re-flushing (with plenty of disk // space) will succeed in flushing the // deletes: fail( testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT); } } else { // On hitting exception we still may have added // all docs: if (result2 != START_COUNT && result2 != END_COUNT) { err.printStackTrace(); fail( testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT); } } searcher.close(); newReader.close(); if (result2 == END_COUNT) { break; } } dir.close(); // Try again with 10 more bytes of free space: diskFree += 10; } }