public void testStallControl() throws InterruptedException, IOException {
int[] numThreads = new int[] {4 + random().nextInt(8), 1};
final int numDocumentsToIndex = 50 + random().nextInt(50);
for (int i = 0; i < numThreads.length; i++) {
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
MockDirectoryWrapper dir = newMockDirectory();
// mock a very slow harddisk sometimes here so that flushing is very slow
dir.setThrottling(MockDirectoryWrapper.Throttling.SOMETIMES);
IndexWriterConfig iwc =
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.setFlushPolicy(flushPolicy);
DocumentsWriterPerThreadPool threadPool =
new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.setIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.setRAMBufferSizeMB(0.25);
IndexWriter writer = new IndexWriter(dir, iwc);
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.length; x++) {
threads[x] = new IndexThread(numDocs, numThreads[i], writer, lineDocFile, false);
threads[x].start();
}
for (int x = 0; x < threads.length; x++) {
threads[x].join();
}
DocumentsWriter docsWriter = writer.getDocsWriter();
assertNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
assertEquals(" all flushes must be due", 0, flushControl.flushBytes());
assertEquals(numDocumentsToIndex, writer.numDocs());
assertEquals(numDocumentsToIndex, writer.maxDoc());
if (numThreads[i] == 1) {
assertFalse(
"single thread must not block numThreads: " + numThreads[i],
docsWriter.flushControl.stallControl.hasBlocked());
}
if (docsWriter.flushControl.peakNetBytes
> (2.d * iwc.getRAMBufferSizeMB() * 1024.d * 1024.d)) {
assertTrue(docsWriter.flushControl.stallControl.wasStalled());
}
assertActiveBytesAfter(flushControl);
writer.close(true);
dir.close();
}
}
public void testRandom() throws IOException, InterruptedException {
final int numThreads = 1 + random().nextInt(8);
final int numDocumentsToIndex = 50 + atLeast(70);
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
iwc.setFlushPolicy(flushPolicy);
final int numDWPT = 1 + random().nextInt(8);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.setIndexerThreadPool(threadPool);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy) writer.getConfig().getFlushPolicy();
DocumentsWriter docsWriter = writer.getDocsWriter();
assertNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes());
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.length; x++) {
threads[x] = new IndexThread(numDocs, numThreads, writer, lineDocFile, true);
threads[x].start();
}
for (int x = 0; x < threads.length; x++) {
threads[x].join();
}
assertEquals(" all flushes must be due", 0, flushControl.flushBytes());
assertEquals(numDocumentsToIndex, writer.numDocs());
assertEquals(numDocumentsToIndex, writer.maxDoc());
if (flushPolicy.flushOnRAM()
&& !flushPolicy.flushOnDocCount()
&& !flushPolicy.flushOnDeleteTerms()) {
final long maxRAMBytes = (long) (iwc.getRAMBufferSizeMB() * 1024. * 1024.);
assertTrue(
"peak bytes without flush exceeded watermark",
flushPolicy.peakBytesWithoutFlush <= maxRAMBytes);
if (flushPolicy.hasMarkedPending) {
assertTrue(
"max: " + maxRAMBytes + " " + flushControl.peakActiveBytes,
maxRAMBytes <= flushControl.peakActiveBytes);
}
}
assertActiveBytesAfter(flushControl);
writer.commit();
assertEquals(0, flushControl.activeBytes());
IndexReader r = DirectoryReader.open(dir);
assertEquals(numDocumentsToIndex, r.numDocs());
assertEquals(numDocumentsToIndex, r.maxDoc());
if (!flushPolicy.flushOnRAM()) {
assertFalse(
"never stall if we don't flush on RAM",
docsWriter.flushControl.stallControl.wasStalled());
assertFalse(
"never block if we don't flush on RAM",
docsWriter.flushControl.stallControl.hasBlocked());
}
r.close();
writer.close();
dir.close();
}
