protected void assertActiveBytesAfter(DocumentsWriterFlushControl flushControl) { Iterator<ThreadState> allActiveThreads = flushControl.allActiveThreadStates(); long bytesUsed = 0; while (allActiveThreads.hasNext()) { bytesUsed += allActiveThreads.next().dwpt.bytesUsed(); } assertEquals(bytesUsed, flushControl.activeBytes()); }
protected void runFlushByRam(int numThreads, double maxRamMB, boolean ensureNotStalled) throws IOException, InterruptedException { final int numDocumentsToIndex = 10 + atLeast(30); AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); Directory dir = newDirectory(); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setFlushPolicy(flushPolicy); final int numDWPT = 1 + atLeast(2); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.setIndexerThreadPool(threadPool); iwc.setRAMBufferSizeMB(maxRamMB); iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy) writer.getConfig().getFlushPolicy(); assertFalse(flushPolicy.flushOnDocCount()); assertFalse(flushPolicy.flushOnDeleteTerms()); assertTrue(flushPolicy.flushOnRAM()); DocumentsWriter docsWriter = writer.getDocsWriter(); assertNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes()); IndexThread[] threads = new IndexThread[numThreads]; for (int x = 0; x < threads.length; x++) { threads[x] = new IndexThread(numDocs, numThreads, writer, lineDocFile, false); threads[x].start(); } for (int x = 0; x < threads.length; x++) { threads[x].join(); } final long maxRAMBytes = (long) (iwc.getRAMBufferSizeMB() * 1024. * 1024.); assertEquals(" all flushes must be due numThreads=" + numThreads, 0, flushControl.flushBytes()); assertEquals(numDocumentsToIndex, writer.numDocs()); assertEquals(numDocumentsToIndex, writer.maxDoc()); assertTrue( "peak bytes without flush exceeded watermark", flushPolicy.peakBytesWithoutFlush <= maxRAMBytes); assertActiveBytesAfter(flushControl); if (flushPolicy.hasMarkedPending) { assertTrue(maxRAMBytes < flushControl.peakActiveBytes); } if (ensureNotStalled) { assertFalse(docsWriter.flushControl.stallControl.wasStalled()); } writer.close(); assertEquals(0, flushControl.activeBytes()); dir.close(); }
public void testFlushDocCount() throws IOException, InterruptedException { int[] numThreads = new int[] {2 + atLeast(1), 1}; for (int i = 0; i < numThreads.length; i++) { final int numDocumentsToIndex = 50 + atLeast(30); AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); Directory dir = newDirectory(); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setFlushPolicy(flushPolicy); final int numDWPT = 1 + atLeast(2); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.setIndexerThreadPool(threadPool); iwc.setMaxBufferedDocs(2 + atLeast(10)); iwc.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy) writer.getConfig().getFlushPolicy(); assertTrue(flushPolicy.flushOnDocCount()); assertFalse(flushPolicy.flushOnDeleteTerms()); assertFalse(flushPolicy.flushOnRAM()); DocumentsWriter docsWriter = writer.getDocsWriter(); assertNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes()); IndexThread[] threads = new IndexThread[numThreads[i]]; for (int x = 0; x < threads.length; x++) { threads[x] = new IndexThread(numDocs, numThreads[i], writer, lineDocFile, false); threads[x].start(); } for (int x = 0; x < threads.length; x++) { threads[x].join(); } assertEquals( " all flushes must be due numThreads=" + numThreads[i], 0, flushControl.flushBytes()); assertEquals(numDocumentsToIndex, writer.numDocs()); assertEquals(numDocumentsToIndex, writer.maxDoc()); assertTrue( "peak bytes without flush exceeded watermark", flushPolicy.peakDocCountWithoutFlush <= iwc.getMaxBufferedDocs()); assertActiveBytesAfter(flushControl); writer.close(); assertEquals(0, flushControl.activeBytes()); dir.close(); } }
@Override public void onInsert(DocumentsWriterFlushControl control, ThreadState state) { final ArrayList<ThreadState> pending = new ArrayList<DocumentsWriterPerThreadPool.ThreadState>(); final ArrayList<ThreadState> notPending = new ArrayList<DocumentsWriterPerThreadPool.ThreadState>(); findPending(control, pending, notPending); final boolean flushCurrent = state.flushPending; long activeBytes = control.activeBytes(); final ThreadState toFlush; if (state.flushPending) { toFlush = state; } else if (flushOnDocCount() && state.dwpt.getNumDocsInRAM() >= indexWriterConfig.getMaxBufferedDocs()) { toFlush = state; } else if (flushOnRAM() && activeBytes >= (long) (indexWriterConfig.getRAMBufferSizeMB() * 1024. * 1024.)) { toFlush = findLargestNonPendingWriter(control, state); assertFalse(toFlush.flushPending); } else { toFlush = null; } super.onInsert(control, state); if (toFlush != null) { if (flushCurrent) { assertTrue(pending.remove(toFlush)); } else { assertTrue(notPending.remove(toFlush)); } assertTrue(toFlush.flushPending); hasMarkedPending = true; } else { peakBytesWithoutFlush = Math.max(activeBytes, peakBytesWithoutFlush); peakDocCountWithoutFlush = Math.max(state.dwpt.getNumDocsInRAM(), peakDocCountWithoutFlush); } for (ThreadState threadState : notPending) { assertFalse(threadState.flushPending); } }
public void testRandom() throws IOException, InterruptedException { final int numThreads = 1 + random().nextInt(8); final int numDocumentsToIndex = 50 + atLeast(70); AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); iwc.setFlushPolicy(flushPolicy); final int numDWPT = 1 + random().nextInt(8); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.setIndexerThreadPool(threadPool); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy) writer.getConfig().getFlushPolicy(); DocumentsWriter docsWriter = writer.getDocsWriter(); assertNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes()); IndexThread[] threads = new IndexThread[numThreads]; for (int x = 0; x < threads.length; x++) { threads[x] = new IndexThread(numDocs, numThreads, writer, lineDocFile, true); threads[x].start(); } for (int x = 0; x < threads.length; x++) { threads[x].join(); } assertEquals(" all flushes must be due", 0, flushControl.flushBytes()); assertEquals(numDocumentsToIndex, writer.numDocs()); assertEquals(numDocumentsToIndex, writer.maxDoc()); if (flushPolicy.flushOnRAM() && !flushPolicy.flushOnDocCount() && !flushPolicy.flushOnDeleteTerms()) { final long maxRAMBytes = (long) (iwc.getRAMBufferSizeMB() * 1024. * 1024.); assertTrue( "peak bytes without flush exceeded watermark", flushPolicy.peakBytesWithoutFlush <= maxRAMBytes); if (flushPolicy.hasMarkedPending) { assertTrue( "max: " + maxRAMBytes + " " + flushControl.peakActiveBytes, maxRAMBytes <= flushControl.peakActiveBytes); } } assertActiveBytesAfter(flushControl); writer.commit(); assertEquals(0, flushControl.activeBytes()); IndexReader r = DirectoryReader.open(dir); assertEquals(numDocumentsToIndex, r.numDocs()); assertEquals(numDocumentsToIndex, r.maxDoc()); if (!flushPolicy.flushOnRAM()) { assertFalse( "never stall if we don't flush on RAM", docsWriter.flushControl.stallControl.wasStalled()); assertFalse( "never block if we don't flush on RAM", docsWriter.flushControl.stallControl.hasBlocked()); } r.close(); writer.close(); dir.close(); }