@Test
public void testMixedChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random());
Store store =
new Store(
shardId,
ImmutableSettings.EMPTY,
directoryService,
randomDistributor(directoryService),
new DummyShardLock(shardId));
// this time random codec....
IndexWriter writer =
new IndexWriter(
store.directory(),
newIndexWriterConfig(random(), new MockAnalyzer(random()))
.setCodec(actualDefaultCodec()));
int docs = 1 + random().nextInt(100);
for (int i = 0; i < docs; i++) {
Document doc = new Document();
doc.add(
new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new SortedDocValuesField(
"dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
writer.addDocument(doc);
}
if (random().nextBoolean()) {
for (int i = 0; i < docs; i++) {
if (random().nextBoolean()) {
Document doc = new Document();
doc.add(
new TextField(
"id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
writer.updateDocument(new Term("id", "" + i), doc);
}
}
}
if (random().nextBoolean()) {
DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
}
Store.MetadataSnapshot metadata;
// check before we committed
try {
store.getMetadata();
fail("no index present - expected exception");
} catch (IndexNotFoundException ex) {
// expected
}
assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed
writer.commit();
writer.close();
Store.LegacyChecksums checksums = new Store.LegacyChecksums();
metadata = store.getMetadata();
assertThat(metadata.asMap().isEmpty(), is(false));
for (StoreFileMetaData meta : metadata) {
try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
if (meta.checksum() == null) {
String checksum = null;
try {
CodecUtil.retrieveChecksum(input);
fail("expected a corrupt index - posting format has not checksums");
} catch (CorruptIndexException
| IndexFormatTooOldException
| IndexFormatTooNewException ex) {
try (ChecksumIndexInput checksumIndexInput =
store.directory().openChecksumInput(meta.name(), IOContext.DEFAULT)) {
checksumIndexInput.seek(meta.length());
checksum = Store.digestToString(checksumIndexInput.getChecksum());
}
// fine - it's a postings format without checksums
checksums.add(new StoreFileMetaData(meta.name(), meta.length(), checksum, null));
}
} else {
String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
assertThat(
"File: " + meta.name() + " has a different checksum",
meta.checksum(),
equalTo(checksum));
assertThat(meta.hasLegacyChecksum(), equalTo(false));
assertThat(meta.writtenBy(), equalTo(Version.LATEST));
}
}
}
assertConsistent(store, metadata);
checksums.write(store);
metadata = store.getMetadata();
assertThat(metadata.asMap().isEmpty(), is(false));
for (StoreFileMetaData meta : metadata) {
assertThat(
"file: " + meta.name() + " has a null checksum", meta.checksum(), not(nullValue()));
if (meta.hasLegacyChecksum()) {
try (ChecksumIndexInput checksumIndexInput =
store.directory().openChecksumInput(meta.name(), IOContext.DEFAULT)) {
checksumIndexInput.seek(meta.length());
assertThat(
meta.checksum(), equalTo(Store.digestToString(checksumIndexInput.getChecksum())));
}
} else {
try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
assertThat(
"File: " + meta.name() + " has a different checksum",
meta.checksum(),
equalTo(checksum));
assertThat(meta.hasLegacyChecksum(), equalTo(false));
assertThat(meta.writtenBy(), equalTo(Version.LATEST));
}
}
}
assertConsistent(store, metadata);
TestUtil.checkIndex(store.directory());
assertDeleteContent(store, directoryService);
IOUtils.close(store);
}
// IF THIS TEST FAILS ON UPGRADE GO LOOK AT THE
// OldSIMockingCodec!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Test
public void testWriteLegacyChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random());
Store store =
new Store(
shardId,
ImmutableSettings.EMPTY,
directoryService,
randomDistributor(directoryService),
new DummyShardLock(shardId));
// set default codec - all segments need checksums
final boolean usesOldCodec = randomBoolean();
IndexWriter writer =
new IndexWriter(
store.directory(),
newIndexWriterConfig(random(), new MockAnalyzer(random()))
.setCodec(usesOldCodec ? new OldSIMockingCodec() : actualDefaultCodec()));
int docs = 1 + random().nextInt(100);
for (int i = 0; i < docs; i++) {
Document doc = new Document();
doc.add(
new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new SortedDocValuesField(
"dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
writer.addDocument(doc);
}
if (random().nextBoolean()) {
for (int i = 0; i < docs; i++) {
if (random().nextBoolean()) {
Document doc = new Document();
doc.add(
new TextField(
"id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
writer.updateDocument(new Term("id", "" + i), doc);
}
}
}
if (random().nextBoolean()) {
DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
}
Store.MetadataSnapshot metadata;
// check before we committed
try {
store.getMetadata();
fail("no index present - expected exception");
} catch (IndexNotFoundException ex) {
// expected
}
assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed
writer.close();
Store.LegacyChecksums checksums = new Store.LegacyChecksums();
Map<String, StoreFileMetaData> legacyMeta = new HashMap<>();
for (String file : store.directory().listAll()) {
if (file.equals("write.lock") || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) {
continue;
}
BytesRef hash = new BytesRef();
if (file.startsWith("segments")) {
hash = Store.MetadataSnapshot.hashFile(store.directory(), file);
}
StoreFileMetaData storeFileMetaData =
new StoreFileMetaData(
file, store.directory().fileLength(file), file + "checksum", null, hash);
legacyMeta.put(file, storeFileMetaData);
checksums.add(storeFileMetaData);
}
checksums.write(store);
metadata = store.getMetadata();
Map<String, StoreFileMetaData> stringStoreFileMetaDataMap = metadata.asMap();
assertThat(legacyMeta.size(), equalTo(stringStoreFileMetaDataMap.size()));
if (usesOldCodec) {
for (StoreFileMetaData meta : legacyMeta.values()) {
assertTrue(meta.toString(), stringStoreFileMetaDataMap.containsKey(meta.name()));
assertEquals(meta.name() + "checksum", meta.checksum());
assertTrue(
meta + " vs. " + stringStoreFileMetaDataMap.get(meta.name()),
stringStoreFileMetaDataMap.get(meta.name()).isSame(meta));
}
} else {
// even if we have a legacy checksum - if we use a new codec we should reuse
for (StoreFileMetaData meta : legacyMeta.values()) {
assertTrue(meta.toString(), stringStoreFileMetaDataMap.containsKey(meta.name()));
assertFalse(
meta + " vs. " + stringStoreFileMetaDataMap.get(meta.name()),
stringStoreFileMetaDataMap.get(meta.name()).isSame(meta));
StoreFileMetaData storeFileMetaData = metadata.get(meta.name());
try (IndexInput input =
store.openVerifyingInput(meta.name(), IOContext.DEFAULT, storeFileMetaData)) {
assertTrue(storeFileMetaData.toString(), input instanceof Store.VerifyingIndexInput);
input.seek(meta.length());
Store.verify(input);
}
}
}
assertDeleteContent(store, directoryService);
IOUtils.close(store);
}
@Test
public void testNewChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random());
Store store =
new Store(
shardId,
ImmutableSettings.EMPTY,
directoryService,
randomDistributor(directoryService),
new DummyShardLock(shardId));
// set default codec - all segments need checksums
IndexWriter writer =
new IndexWriter(
store.directory(),
newIndexWriterConfig(random(), new MockAnalyzer(random()))
.setCodec(actualDefaultCodec()));
int docs = 1 + random().nextInt(100);
for (int i = 0; i < docs; i++) {
Document doc = new Document();
doc.add(
new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new SortedDocValuesField(
"dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
writer.addDocument(doc);
}
if (random().nextBoolean()) {
for (int i = 0; i < docs; i++) {
if (random().nextBoolean()) {
Document doc = new Document();
doc.add(
new TextField(
"id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
writer.updateDocument(new Term("id", "" + i), doc);
}
}
}
if (random().nextBoolean()) {
DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
}
Store.MetadataSnapshot metadata;
// check before we committed
try {
store.getMetadata();
fail("no index present - expected exception");
} catch (IndexNotFoundException ex) {
// expected
}
assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed
writer.commit();
writer.close();
metadata = store.getMetadata();
assertThat(metadata.asMap().isEmpty(), is(false));
for (StoreFileMetaData meta : metadata) {
try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
assertThat(
"File: " + meta.name() + " has a different checksum",
meta.checksum(),
equalTo(checksum));
assertThat(meta.hasLegacyChecksum(), equalTo(false));
assertThat(meta.writtenBy(), equalTo(Version.LATEST));
if (meta.name().endsWith(".si") || meta.name().startsWith("segments_")) {
assertThat(meta.hash().length, greaterThan(0));
}
}
}
assertConsistent(store, metadata);
TestUtil.checkIndex(store.directory());
assertDeleteContent(store, directoryService);
IOUtils.close(store);
}
@Test
public void testCleanupFromSnapshot() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random());
Store store =
new Store(
shardId,
ImmutableSettings.EMPTY,
directoryService,
randomDistributor(directoryService),
new DummyShardLock(shardId));
// this time random codec....
IndexWriterConfig indexWriterConfig =
newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(actualDefaultCodec());
// we keep all commits and that allows us clean based on multiple snapshots
indexWriterConfig.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
IndexWriter writer = new IndexWriter(store.directory(), indexWriterConfig);
int docs = 1 + random().nextInt(100);
int numCommits = 0;
for (int i = 0; i < docs; i++) {
if (i > 0 && randomIntBetween(0, 10) == 0) {
writer.commit();
numCommits++;
}
Document doc = new Document();
doc.add(
new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new SortedDocValuesField(
"dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
writer.addDocument(doc);
}
if (numCommits < 1) {
writer.commit();
Document doc = new Document();
doc.add(
new TextField(
"id", "" + docs++, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new SortedDocValuesField(
"dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
writer.addDocument(doc);
}
Store.MetadataSnapshot firstMeta = store.getMetadata();
if (random().nextBoolean()) {
for (int i = 0; i < docs; i++) {
if (random().nextBoolean()) {
Document doc = new Document();
doc.add(
new TextField(
"id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
writer.updateDocument(new Term("id", "" + i), doc);
}
}
}
writer.commit();
writer.close();
Store.MetadataSnapshot secondMeta = store.getMetadata();
Store.LegacyChecksums checksums = new Store.LegacyChecksums();
Map<String, StoreFileMetaData> legacyMeta = new HashMap<>();
for (String file : store.directory().listAll()) {
if (file.equals("write.lock") || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) {
continue;
}
BytesRef hash = new BytesRef();
if (file.startsWith("segments")) {
hash = Store.MetadataSnapshot.hashFile(store.directory(), file);
}
StoreFileMetaData storeFileMetaData =
new StoreFileMetaData(
file, store.directory().fileLength(file), file + "checksum", null, hash);
legacyMeta.put(file, storeFileMetaData);
checksums.add(storeFileMetaData);
}
checksums.write(
store); // write one checksum file here - we expect it to survive all the cleanups
if (randomBoolean()) {
store.cleanupAndVerify("test", firstMeta);
String[] strings = store.directory().listAll();
int numChecksums = 0;
int numNotFound = 0;
for (String file : strings) {
assertTrue(firstMeta.contains(file) || Store.isChecksum(file));
if (Store.isChecksum(file)) {
numChecksums++;
} else if (secondMeta.contains(file) == false) {
numNotFound++;
}
}
assertTrue(
"at least one file must not be in here since we have two commits?", numNotFound > 0);
assertEquals(
"we wrote one checksum but it's gone now? - checksums are supposed to be kept",
numChecksums,
1);
} else {
store.cleanupAndVerify("test", secondMeta);
String[] strings = store.directory().listAll();
int numChecksums = 0;
int numNotFound = 0;
for (String file : strings) {
assertTrue(secondMeta.contains(file) || Store.isChecksum(file));
if (Store.isChecksum(file)) {
numChecksums++;
} else if (firstMeta.contains(file) == false) {
numNotFound++;
}
}
assertTrue(
"at least one file must not be in here since we have two commits?", numNotFound > 0);
assertEquals(
"we wrote one checksum but it's gone now? - checksums are supposed to be kept",
numChecksums,
1);
}
store.deleteContent();
IOUtils.close(store);
}