/** Generate a spelling suggestion for the definitions stored in defs */
public void createSpellingSuggestions() {
IndexReader indexReader = null;
SpellChecker checker;
try {
log.info("Generating spelling suggestion index ... ");
indexReader = DirectoryReader.open(indexDirectory);
checker = new SpellChecker(spellDirectory);
// TODO below seems only to index "defs" , possible bug ?
Analyzer analyzer = AnalyzerGuru.getAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
checker.indexDictionary(new LuceneDictionary(indexReader, QueryBuilder.DEFS), iwc, false);
log.info("done");
} catch (IOException e) {
log.log(Level.SEVERE, "ERROR: Generating spelling: {0}", e);
} finally {
if (indexReader != null) {
try {
indexReader.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing reader", e);
}
}
if (spellDirectory != null) {
spellDirectory.close();
}
}
}
IndexWriter getIndexWriter(Directory luceneDir)
throws CorruptIndexException, LockObtainFailedException, IOException, ProviderException {
IndexWriter writer = null;
IndexWriterConfig writerConfig = new IndexWriterConfig(Version.LUCENE_36, getLuceneAnalyzer());
writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
writer = new IndexWriter(luceneDir, writerConfig);
// writer.setInfoStream( System.out );
return writer;
}
protected DirectoryReader indexDocsWithLucene(TestDoc[] testDocs) throws IOException {
Map<String, Analyzer> mapping = new HashMap<String, Analyzer>();
for (TestFieldSetting field : testDocs[0].fieldSettings) {
if (field.storedPayloads) {
mapping.put(
field.name,
new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new StandardTokenizer(Version.CURRENT.luceneVersion, reader);
TokenFilter filter = new LowerCaseFilter(Version.CURRENT.luceneVersion, tokenizer);
filter = new TypeAsPayloadTokenFilter(filter);
return new TokenStreamComponents(tokenizer, filter);
}
});
}
}
PerFieldAnalyzerWrapper wrapper =
new PerFieldAnalyzerWrapper(
new StandardAnalyzer(Version.CURRENT.luceneVersion, CharArraySet.EMPTY_SET), mapping);
Directory dir = new RAMDirectory();
IndexWriterConfig conf = new IndexWriterConfig(Version.CURRENT.luceneVersion, wrapper);
conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
IndexWriter writer = new IndexWriter(dir, conf);
for (TestDoc doc : testDocs) {
Document d = new Document();
d.add(new Field("id", doc.id, StringField.TYPE_STORED));
for (int i = 0; i < doc.fieldContent.length; i++) {
FieldType type = new FieldType(TextField.TYPE_STORED);
TestFieldSetting fieldSetting = doc.fieldSettings[i];
type.setStoreTermVectorOffsets(fieldSetting.storedOffset);
type.setStoreTermVectorPayloads(fieldSetting.storedPayloads);
type.setStoreTermVectorPositions(
fieldSetting.storedPositions
|| fieldSetting.storedPayloads
|| fieldSetting.storedOffset);
type.setStoreTermVectors(true);
type.freeze();
d.add(new Field(fieldSetting.name, doc.fieldContent[i], type));
}
writer.updateDocument(new Term("id", doc.id), d);
writer.commit();
}
writer.close();
return DirectoryReader.open(dir);
}
/**
* Index a picture
*
* @param source
* @param picture_id
* @param conf
* @throws IOException
*/
public static void index(byte[] source, UUID picture_id, IndexWriterConfig conf)
throws IOException {
ByteArrayInputStream in = new ByteArrayInputStream(source);
BufferedImage image = ImageIO.read(in);
// Creating an Lucene IndexWriter
log.debug("Is Lucene configured? " + (conf == null));
if (conf == null) {
conf =
new IndexWriterConfig(
LuceneUtils.LUCENE_VERSION, new WhitespaceAnalyzer(LuceneUtils.LUCENE_VERSION));
conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
}
luceneIndexer(
image,
picture_id,
FeatureEnumerate.AutoColorCorrelogram.getText(),
DocumentBuilderFactory.getAutoColorCorrelogramDocumentBuilder(),
conf);
luceneIndexer(
image,
picture_id,
FeatureEnumerate.CEDD.getText(),
DocumentBuilderFactory.getCEDDDocumentBuilder(),
conf);
luceneIndexer(
image,
picture_id,
FeatureEnumerate.ColorLayout.getText(),
DocumentBuilderFactory.getColorLayoutBuilder(),
conf);
luceneIndexer(
image,
picture_id,
FeatureEnumerate.EdgeHistogram.getText(),
DocumentBuilderFactory.getEdgeHistogramBuilder(),
conf);
luceneIndexer(
image,
picture_id,
FeatureEnumerate.ColorHistogram.getText(),
DocumentBuilderFactory.getColorHistogramDocumentBuilder(),
conf);
luceneIndexer(
image,
picture_id,
FeatureEnumerate.PHOG.getText(),
DocumentBuilderFactory.getPHOGDocumentBuilder(),
conf);
}
public void testTypeChangeAfterOpenCreate() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
Document doc = new Document();
doc.add(new NumericDocValuesField("dv", 0L));
writer.addDocument(doc);
writer.close();
conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
writer = new IndexWriter(dir, conf);
doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
writer.addDocument(doc);
writer.close();
dir.close();
}
private static void deleteFromFeature(
UUID pictureId, Term term, String prefix, IndexWriterConfig conf) throws IOException {
File file = getPath(prefix);
// Creating an Lucene IndexWriter
log.debug("Is Lucene configured: " + (conf == null));
if (conf == null) {
conf =
new IndexWriterConfig(
LuceneUtils.LUCENE_VERSION, new WhitespaceAnalyzer(LuceneUtils.LUCENE_VERSION));
conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
}
IndexWriter iw = new IndexWriter(FSDirectory.open(file), conf);
iw.deleteDocuments(term);
iw.close();
}
// Add inconsistent document after reopening IW w/ create
public void testMixedTypesAfterReopenCreate() throws Exception {
Directory dir = newDirectory();
IndexWriter w =
new IndexWriter(
dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document doc = new Document();
doc.add(new NumericDocValuesField("foo", 0));
w.addDocument(doc);
w.close();
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
w = new IndexWriter(dir, iwc);
doc = new Document();
doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
w.addDocument(doc);
w.close();
dir.close();
}
/** Optimize the index database */
public void optimize() {
synchronized (lock) {
if (running) {
log.warning("Optimize terminated... Someone else is updating / optimizing it!");
return;
}
running = true;
}
IndexWriter wrt = null;
try {
log.info("Optimizing the index ... ");
Analyzer analyzer = new StandardAnalyzer(SearchEngine.LUCENE_VERSION);
IndexWriterConfig conf = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
conf.setOpenMode(OpenMode.CREATE_OR_APPEND);
wrt = new IndexWriter(indexDirectory, conf);
wrt.forceMerge(1); // this is deprecated and not needed anymore
log.info("done");
synchronized (lock) {
if (dirtyFile.exists() && !dirtyFile.delete()) {
log.log(Level.FINE, "Failed to remove \"dirty-file\": {0}", dirtyFile.getAbsolutePath());
}
dirty = false;
}
} catch (IOException e) {
log.log(Level.SEVERE, "ERROR: optimizing index: {0}", e);
} finally {
if (wrt != null) {
try {
wrt.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing writer", e);
}
}
synchronized (lock) {
running = false;
}
}
}
/**
* Update the content of this index database
*
* @throws IOException if an error occurs
* @throws HistoryException if an error occurs when accessing the history
*/
public void update() throws IOException, HistoryException {
synchronized (lock) {
if (running) {
throw new IOException("Indexer already running!");
}
running = true;
interrupted = false;
}
String ctgs = RuntimeEnvironment.getInstance().getCtags();
if (ctgs != null) {
ctags = new Ctags();
ctags.setBinary(ctgs);
}
if (ctags == null) {
log.severe("Unable to run ctags! searching definitions will not work!");
}
if (ctags != null) {
String filename = RuntimeEnvironment.getInstance().getCTagsExtraOptionsFile();
if (filename != null) {
ctags.setCTagsExtraOptionsFile(filename);
}
}
try {
Analyzer analyzer = AnalyzerGuru.getAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
// iwc.setRAMBufferSizeMB(256.0); //TODO check what is the sweet spot
writer = new IndexWriter(indexDirectory, iwc);
writer.commit(); // to make sure index exists on the disk
// writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
if (directories.isEmpty()) {
if (project == null) {
directories.add("");
} else {
directories.add(project.getPath());
}
}
for (String dir : directories) {
File sourceRoot;
if ("".equals(dir)) {
sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
} else {
sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
}
HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
String startuid = Util.path2uid(dir, "");
IndexReader reader = DirectoryReader.open(indexDirectory); // open existing index
Terms terms = null;
int numDocs = reader.numDocs();
if (numDocs > 0) {
Fields uFields = MultiFields.getFields(reader); // reader.getTermVectors(0);
terms = uFields.terms(QueryBuilder.U);
}
try {
if (numDocs > 0) {
uidIter = terms.iterator(null);
TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid), true); // init uid
if (stat == TermsEnum.SeekStatus.END || stat == TermsEnum.SeekStatus.NOT_FOUND) {
uidIter = null;
}
}
// TODO below should be optional, since it traverses the tree once more to get total
// count! :(
int file_cnt = 0;
if (RuntimeEnvironment.getInstance().isPrintProgress()) {
log.log(Level.INFO, "Counting files in {0} ...", dir);
file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
if (log.isLoggable(Level.INFO)) {
log.log(
Level.INFO, "Need to process: {0} files for {1}", new Object[] {file_cnt, dir});
}
}
indexDown(sourceRoot, dir, false, 0, file_cnt);
while (uidIter != null
&& uidIter.term() != null
&& uidIter.term().utf8ToString().startsWith(startuid)) {
removeFile();
uidIter.next();
}
} finally {
reader.close();
}
}
} finally {
if (writer != null) {
try {
writer.prepareCommit();
writer.commit();
writer.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing writer", e);
}
}
if (ctags != null) {
try {
ctags.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing ctags process", e);
}
}
synchronized (lock) {
running = false;
}
}
if (!isInterrupted() && isDirty()) {
if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
optimize();
}
createSpellingSuggestions();
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File timestamp = new File(env.getDataRootFile(), "timestamp");
if (timestamp.exists()) {
if (!timestamp.setLastModified(System.currentTimeMillis())) {
log.log(
Level.WARNING,
"Failed to set last modified time on ''{0}'', used for timestamping the index database.",
timestamp.getAbsolutePath());
}
} else {
if (!timestamp.createNewFile()) {
log.log(
Level.WARNING,
"Failed to create file ''{0}'', used for timestamping the index database.",
timestamp.getAbsolutePath());
}
}
}
}
public void createSenIndex(JSONArray jsonArray, String indexPath, String stopwordsFile)
throws Exception {
if (jsonArray == null) {
System.out.println("error: jsonArray is null!\n");
return;
}
Analyzer analyzer = null;
if (stopwordsFile == null) {
analyzer = new SimpleAnalyzer();
} else {
analyzer = new StopAnalyzer(Paths.get(stopwordsFile));
}
IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
IndexWriter indexWriter = new IndexWriter(FSDirectory.open(Paths.get(indexPath)), iwc);
indexWriter.deleteAll();
TextUtil textUtil = new TextUtil();
long startTime = new Date().getTime();
System.out.println("jsonArray size: " + jsonArray.size());
long num_sentence = 0;
for (JSONObject jsonObj : (List<JSONObject>) jsonArray) {
long id = (long) jsonObj.get(idKey);
String review = (String) jsonObj.get(reviewKey);
if (review == null || review.isEmpty()) {
continue;
}
if (review.matches(".*[^\\x00-\\x7F].*")) {
continue;
}
String[] tokens = textUtil.tokenize(review);
if (tokens.length <= TERM_MIN_THRESHOLD) {
continue;
}
String[] sentences = textUtil.sentenceDetect(review);
// System.out.println(body.toLowerCase() + "\n");
num_sentence = 0;
for (int i = 0; i < sentences.length; i++) {
if (sentences[i] == null || sentences[i].isEmpty()) {
continue;
}
// System.out.println(sentences[i]);
Document doc = new Document();
Field idField = new LongField(idKey, id, Field.Store.YES);
Field numField = new LongField("num", num_sentence, Field.Store.NO);
Field contentField =
new TextField(
reviewKey,
sentences[i].replaceAll("[_'.,]", " ").replaceAll("[0-9]", ""),
Field.Store.YES);
doc.add(idField);
doc.add(numField);
doc.add(contentField);
indexWriter.addDocument(doc);
num_sentence++;
}
}
indexWriter.commit();
indexWriter.close();
long endTime = new Date().getTime();
System.out.println("\n\ncreate index time: " + (endTime - startTime) + "ms");
System.out.println("\n sentence num: " + num_sentence + "\n");
}
@Nightly
public void test() throws Exception {
MockDirectoryWrapper dir =
new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("4GBStoredFields")));
dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.setRAMBufferSizeMB(256.0);
iwc.setMergeScheduler(new ConcurrentMergeScheduler());
iwc.setMergePolicy(newLogMergePolicy(false, 10));
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
// TODO: we disable "Compressing" since it likes to pick very extreme values which will be too
// slow for this test.
// maybe we should factor out crazy cases to ExtremeCompressing? then annotations can handle
// this stuff...
if (random().nextBoolean()) {
iwc.setCodec(CompressingCodec.reasonableInstance(random()));
}
IndexWriter w = new IndexWriter(dir, iwc);
MergePolicy mp = w.getConfig().getMergePolicy();
if (mp instanceof LogByteSizeMergePolicy) {
// 1 petabyte:
((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
}
final Document doc = new Document();
final FieldType ft = new FieldType();
ft.setStored(true);
ft.freeze();
final int valueLength = RandomInts.randomIntBetween(random(), 1 << 13, 1 << 20);
final byte[] value = new byte[valueLength];
for (int i = 0; i < valueLength; ++i) {
// random so that even compressing codecs can't compress it
value[i] = (byte) random().nextInt(256);
}
final Field f = new Field("fld", value, ft);
doc.add(f);
final int numDocs = (int) ((1L << 32) / valueLength + 100);
for (int i = 0; i < numDocs; ++i) {
w.addDocument(doc);
if (VERBOSE && i % (numDocs / 10) == 0) {
System.out.println(i + " of " + numDocs + "...");
}
}
w.forceMerge(1);
w.close();
if (VERBOSE) {
boolean found = false;
for (String file : dir.listAll()) {
if (file.endsWith(".fdt")) {
final long fileLength = dir.fileLength(file);
if (fileLength >= 1L << 32) {
found = true;
}
System.out.println("File length of " + file + " : " + fileLength);
}
}
if (!found) {
System.out.println("No .fdt file larger than 4GB, test bug?");
}
}
DirectoryReader rd = DirectoryReader.open(dir);
Document sd = rd.document(numDocs - 1);
assertNotNull(sd);
assertEquals(1, sd.getFields().size());
BytesRef valueRef = sd.getBinaryValue("fld");
assertNotNull(valueRef);
assertEquals(new BytesRef(value), valueRef);
rd.close();
dir.close();
}
@Test
public void testRecoveryDiff() throws IOException, InterruptedException {
int numDocs = 2 + random().nextInt(100);
List<Document> docs = new ArrayList<>();
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
doc.add(
new StringField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new SortedDocValuesField(
"dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
docs.add(doc);
}
long seed = random().nextLong();
Store.MetadataSnapshot first;
{
Random random = new Random(seed);
IndexWriterConfig iwc =
new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setUseCompoundFile(random.nextBoolean());
iwc.setMaxThreadStates(1);
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random);
Store store =
new Store(
shardId,
ImmutableSettings.EMPTY,
directoryService,
randomDistributor(random, directoryService),
new DummyShardLock(shardId));
IndexWriter writer = new IndexWriter(store.directory(), iwc);
final boolean lotsOfSegments = rarely(random);
for (Document d : docs) {
writer.addDocument(d);
if (lotsOfSegments && random.nextBoolean()) {
writer.commit();
} else if (rarely(random)) {
writer.commit();
}
}
writer.commit();
writer.close();
first = store.getMetadata();
assertDeleteContent(store, directoryService);
store.close();
}
long time = new Date().getTime();
while (time == new Date().getTime()) {
Thread.sleep(10); // bump the time
}
Store.MetadataSnapshot second;
Store store;
{
Random random = new Random(seed);
IndexWriterConfig iwc =
new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setUseCompoundFile(random.nextBoolean());
iwc.setMaxThreadStates(1);
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random);
store =
new Store(
shardId,
ImmutableSettings.EMPTY,
directoryService,
randomDistributor(random, directoryService),
new DummyShardLock(shardId));
IndexWriter writer = new IndexWriter(store.directory(), iwc);
final boolean lotsOfSegments = rarely(random);
for (Document d : docs) {
writer.addDocument(d);
if (lotsOfSegments && random.nextBoolean()) {
writer.commit();
} else if (rarely(random)) {
writer.commit();
}
}
writer.commit();
writer.close();
second = store.getMetadata();
}
Store.RecoveryDiff diff = first.recoveryDiff(second);
assertThat(first.size(), equalTo(second.size()));
for (StoreFileMetaData md : first) {
assertThat(second.get(md.name()), notNullValue());
// si files are different - containing timestamps etc
assertThat(second.get(md.name()).isSame(md), equalTo(false));
}
assertThat(diff.different.size(), equalTo(first.size()));
assertThat(
diff.identical.size(),
equalTo(0)); // in lucene 5 nothing is identical - we use random ids in file headers
assertThat(diff.missing, empty());
// check the self diff
Store.RecoveryDiff selfDiff = first.recoveryDiff(first);
assertThat(selfDiff.identical.size(), equalTo(first.size()));
assertThat(selfDiff.different, empty());
assertThat(selfDiff.missing, empty());
// lets add some deletes
Random random = new Random(seed);
IndexWriterConfig iwc =
new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setUseCompoundFile(random.nextBoolean());
iwc.setMaxThreadStates(1);
iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
IndexWriter writer = new IndexWriter(store.directory(), iwc);
writer.deleteDocuments(new Term("id", Integer.toString(random().nextInt(numDocs))));
writer.commit();
writer.close();
Store.MetadataSnapshot metadata = store.getMetadata();
StoreFileMetaData delFile = null;
for (StoreFileMetaData md : metadata) {
if (md.name().endsWith(".liv")) {
delFile = md;
break;
}
}
Store.RecoveryDiff afterDeleteDiff = metadata.recoveryDiff(second);
if (delFile != null) {
assertThat(
afterDeleteDiff.identical.size(), equalTo(metadata.size() - 2)); // segments_N + del file
assertThat(afterDeleteDiff.different.size(), equalTo(0));
assertThat(afterDeleteDiff.missing.size(), equalTo(2));
} else {
// an entire segment must be missing (single doc segment got dropped)
assertThat(afterDeleteDiff.identical.size(), greaterThan(0));
assertThat(afterDeleteDiff.different.size(), equalTo(0));
assertThat(afterDeleteDiff.missing.size(), equalTo(1)); // the commit file is different
}
// check the self diff
selfDiff = metadata.recoveryDiff(metadata);
assertThat(selfDiff.identical.size(), equalTo(metadata.size()));
assertThat(selfDiff.different, empty());
assertThat(selfDiff.missing, empty());
// add a new commit
iwc = new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setUseCompoundFile(
true); // force CFS - easier to test here since we know it will add 3 files
iwc.setMaxThreadStates(1);
iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
writer = new IndexWriter(store.directory(), iwc);
writer.addDocument(docs.get(0));
writer.close();
Store.MetadataSnapshot newCommitMetaData = store.getMetadata();
Store.RecoveryDiff newCommitDiff = newCommitMetaData.recoveryDiff(metadata);
if (delFile != null) {
assertThat(
newCommitDiff.identical.size(),
equalTo(
newCommitMetaData.size()
- 5)); // segments_N, del file, cfs, cfe, si for the new segment
assertThat(newCommitDiff.different.size(), equalTo(1)); // the del file must be different
assertThat(newCommitDiff.different.get(0).name(), endsWith(".liv"));
assertThat(
newCommitDiff.missing.size(), equalTo(4)); // segments_N,cfs, cfe, si for the new segment
} else {
assertThat(
newCommitDiff.identical.size(),
equalTo(newCommitMetaData.size() - 4)); // segments_N, cfs, cfe, si for the new segment
assertThat(newCommitDiff.different.size(), equalTo(0));
assertThat(
newCommitDiff.missing.size(),
equalTo(
4)); // an entire segment must be missing (single doc segment got dropped) plus the
// commit is different
}
store.deleteContent();
IOUtils.close(store);
}