/*
* Verify that a writer with "commit on close" indeed
* cleans up the temp segments created after opening
* that are not referenced by the starting segments
* file. We check this by using MockDirectoryWrapper to
* measure max temp disk space used.
*/
public void testCommitOnCloseDiskUsage() throws IOException {
// MemoryCodec, since it uses FST, is not necessarily
// "additive", ie if you add up N small FSTs, then merge
// them, the merged result can easily be larger than the
// sum because the merged FST may use array encoding for
// some arcs (which uses more space):
final String idFormat = _TestUtil.getPostingsFormat("id");
final String contentFormat = _TestUtil.getPostingsFormat("content");
assumeFalse(
"This test cannot run with Memory codec",
idFormat.equals("Memory") || contentFormat.equals("Memory"));
MockDirectoryWrapper dir = newDirectory();
Analyzer analyzer;
if (random().nextBoolean()) {
// no payloads
analyzer =
new Analyzer() {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
return new TokenStreamComponents(
new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
}
};
} else {
// fixed length payloads
final int length = random().nextInt(200);
analyzer =
new Analyzer() {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(
tokenizer, new MockFixedLengthPayloadFilter(random(), tokenizer, length));
}
};
}
IndexWriter writer =
new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)
.setMaxBufferedDocs(10)
.setReaderPooling(false)
.setMergePolicy(newLogMergePolicy(10)));
for (int j = 0; j < 30; j++) {
TestIndexWriter.addDocWithIndex(writer, j);
}
writer.close();
dir.resetMaxUsedSizeInBytes();
dir.setTrackDiskUsage(true);
long startDiskUsage = dir.getMaxUsedSizeInBytes();
writer =
new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)
.setOpenMode(OpenMode.APPEND)
.setMaxBufferedDocs(10)
.setMergeScheduler(new SerialMergeScheduler())
.setReaderPooling(false)
.setMergePolicy(newLogMergePolicy(10)));
for (int j = 0; j < 1470; j++) {
TestIndexWriter.addDocWithIndex(writer, j);
}
long midDiskUsage = dir.getMaxUsedSizeInBytes();
dir.resetMaxUsedSizeInBytes();
writer.forceMerge(1);
writer.close();
DirectoryReader.open(dir).close();
long endDiskUsage = dir.getMaxUsedSizeInBytes();
// Ending index is 50X as large as starting index; due
// to 3X disk usage normally we allow 150X max
// transient usage. If something is wrong w/ deleter
// and it doesn't delete intermediate segments then it
// will exceed this 150X:
// System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " +
// endDiskUsage);
assertTrue(
"writer used too much space while adding documents: mid="
+ midDiskUsage
+ " start="
+ startDiskUsage
+ " end="
+ endDiskUsage
+ " max="
+ (startDiskUsage * 150),
midDiskUsage < 150 * startDiskUsage);
assertTrue(
"writer used too much space after close: endDiskUsage="
+ endDiskUsage
+ " startDiskUsage="
+ startDiskUsage
+ " max="
+ (startDiskUsage * 150),
endDiskUsage < 150 * startDiskUsage);
dir.close();
}
