@Test
public void testRemoveUnfinishedLeftovers_abort_multipleFolders() throws Throwable {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
Files.createDirectories(dataFolder2.toPath());
SSTableReader[] sstables = {
sstable(dataFolder1, cfs, 0, 128),
sstable(dataFolder1, cfs, 1, 128),
sstable(dataFolder2, cfs, 2, 128),
sstable(dataFolder2, cfs, 3, 128)
};
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
LogTransaction.SSTableTidier[] tidiers = {
log.obsoleted(sstables[0]), log.obsoleted(sstables[2])
};
log.trackNew(sstables[1]);
log.trackNew(sstables[3]);
Collection<File> logFiles = log.logFiles();
Assert.assertEquals(2, logFiles.size());
// fake an abort
log.txnFile().abort();
Arrays.stream(sstables).forEach(s -> s.selfRef().release());
// test listing
Assert.assertEquals(
sstables[1].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()),
getTemporaryFiles(dataFolder1));
Assert.assertEquals(
sstables[3].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()),
getTemporaryFiles(dataFolder2));
// normally called at startup
LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2));
// old tables should be only table left
assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[0].getAllFilePaths()));
assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[2].getAllFilePaths()));
// complete the transaction to avoid LEAK errors
Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
assertNull(log.complete(null));
}
@Test
public void testCommitSameDesc() throws Throwable {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
SSTableReader sstableOld1 = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableOld2 = sstable(dataFolder, cfs, 0, 256);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
log.trackNew(sstableNew);
sstableOld1.setReplaced();
LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld2);
assertNotNull(tidier);
log.finish();
sstableOld2.markObsolete(tidier);
sstableOld1.selfRef().release();
sstableOld2.selfRef().release();
assertFiles(dataFolder.getPath(), new HashSet<>(sstableNew.getAllFilePaths()));
sstableNew.selfRef().release();
}
@Test
public void testGetTemporaryFilesThrowsIfCompletingAfterObsoletion() throws Throwable {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
assertNotNull(logs);
LogTransaction.SSTableTidier tidier = logs.obsoleted(sstable);
sstable.markObsolete(tidier);
sstable.selfRef().release();
LogTransaction.waitForDeletions();
try {
// This should race with the asynchronous deletion of txn log files
// it should throw because we are violating the requirement that a transaction must
// finish before deleting files (i.e. releasing sstables)
getTemporaryFiles(dataFolder);
fail("Expected runtime exception");
} catch (RuntimeException e) {
// pass as long as the cause is not an assertion
assertFalse(e.getCause() instanceof AssertionError);
}
logs.finish();
}
private static void testCorruptRecord(
BiConsumer<LogTransaction, SSTableReader> modifier, boolean isRecoverable)
throws IOException {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
// simulate tracking sstables with a committed transaction except the checksum will be wrong
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
log.trackNew(sstableNew);
log.obsoleted(sstableOld);
// Modify the transaction log or disk state for sstableOld
modifier.accept(log, sstableOld);
assertNull(log.complete(null));
sstableOld.selfRef().release();
sstableNew.selfRef().release();
// The files on disk, for old files make sure to exclude the files that were deleted by the
// modifier
Set<String> newFiles = sstableNew.getAllFilePaths().stream().collect(Collectors.toSet());
Set<String> oldFiles =
sstableOld
.getAllFilePaths()
.stream()
.filter(p -> new File(p).exists())
.collect(Collectors.toSet());
// This should filter as in progress since the last record is corrupt
assertFiles(newFiles, getTemporaryFiles(dataFolder));
assertFiles(oldFiles, getFinalFiles(dataFolder));
if (isRecoverable) { // the corruption is recoverable but the commit record is unreadable so the
// transaction is still in progress
// This should remove new files
LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
// make sure to exclude the old files that were deleted by the modifier
assertFiles(dataFolder.getPath(), oldFiles);
} else { // if an intermediate line was also modified, it should ignore the tx log file
// This should not remove any files
LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
assertFiles(
dataFolder.getPath(),
Sets.newHashSet(Iterables.concat(newFiles, oldFiles, log.logFilePaths())));
}
}
@Test
public void testCommitMultipleFolders() throws Throwable {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
Files.createDirectories(dataFolder2.toPath());
SSTableReader[] sstables = {
sstable(dataFolder1, cfs, 0, 128),
sstable(dataFolder1, cfs, 1, 128),
sstable(dataFolder2, cfs, 2, 128),
sstable(dataFolder2, cfs, 3, 128)
};
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
LogTransaction.SSTableTidier[] tidiers = {
log.obsoleted(sstables[0]), log.obsoleted(sstables[2])
};
log.trackNew(sstables[1]);
log.trackNew(sstables[3]);
log.finish();
sstables[0].markObsolete(tidiers[0]);
sstables[2].markObsolete(tidiers[1]);
Arrays.stream(sstables).forEach(s -> s.selfRef().release());
LogTransaction.waitForDeletions();
assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[1].getAllFilePaths()));
assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[3].getAllFilePaths()));
}
Transaction(ColumnFamilyStore cfs, LogTransaction txnLogs) throws IOException {
this.cfs = cfs;
this.txnLogs = txnLogs;
this.dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
this.sstableOld = sstable(dataFolder, cfs, 0, 128);
this.sstableNew = sstable(dataFolder, cfs, 1, 128);
assertNotNull(txnLogs);
assertNotNull(txnLogs.id());
Assert.assertEquals(OperationType.COMPACTION, txnLogs.type());
txnLogs.trackNew(sstableNew);
tidier = txnLogs.obsoleted(sstableOld);
assertNotNull(tidier);
}
@Test
public void testCommitOnlyOld() throws Throwable {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
LogTransaction.SSTableTidier tidier = log.obsoleted(sstable);
assertNotNull(tidier);
log.finish();
sstable.markObsolete(tidier);
sstable.selfRef().release();
assertFiles(dataFolder.getPath(), new HashSet<>());
}
private static void testObsoletedFilesChanged(Consumer<SSTableReader> modifier)
throws IOException {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
// simulate tracking sstables with a committed transaction except the checksum will be wrong
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
log.trackNew(sstableNew);
/*TransactionLog.SSTableTidier tidier =*/ log.obsoleted(sstableOld);
// modify the old sstable files
modifier.accept(sstableOld);
// Fake a commit
log.txnFile().commit();
// This should not remove the old files
LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
assertFiles(
dataFolder.getPath(),
Sets.newHashSet(
Iterables.concat(
sstableNew.getAllFilePaths(), sstableOld.getAllFilePaths(), log.logFilePaths())));
sstableOld.selfRef().release();
sstableNew.selfRef().release();
// complete the transaction to avoid LEAK errors
assertNull(log.complete(null));
assertFiles(
dataFolder.getPath(),
Sets.newHashSet(
Iterables.concat(
sstableNew.getAllFilePaths(), sstableOld.getAllFilePaths(), log.logFilePaths())));
}
@Test
public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
assertNotNull(logs);
LogTransaction.SSTableTidier tidier = logs.obsoleted(sstable);
logs.finish();
sstable.markObsolete(tidier);
sstable.selfRef().release();
// This should race with the asynchronous deletion of txn log files
// It doesn't matter what it returns but it should not throw because the txn
// was completed before deleting files (i.e. releasing sstables)
for (int i = 0; i < 200; i++) getTemporaryFiles(dataFolder);
}
@Test
public void testRemoveUnfinishedLeftovers_commit() throws Throwable {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
// simulate tracking sstables with a committed transaction (new log file deleted)
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
log.trackNew(sstableNew);
LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
// Fake a commit
log.txnFile().commit();
Set<File> tmpFiles =
sstableOld.getAllFilePaths().stream().map(File::new).collect(Collectors.toSet());
sstableNew.selfRef().release();
sstableOld.selfRef().release();
Assert.assertEquals(tmpFiles, getTemporaryFiles(sstableOld.descriptor.directory));
// normally called at startup
LogTransaction.removeUnfinishedLeftovers(cfs.metadata);
// sstableNew should be only table left
Directories directories = new Directories(cfs.metadata);
Map<Descriptor, Set<Component>> sstables =
directories.sstableLister(Directories.OnTxnErr.THROW).list();
assertEquals(1, sstables.size());
assertFiles(dataFolder.getPath(), new HashSet<>(sstableNew.getAllFilePaths()));
// complete the transaction to avoid LEAK errors
tidier.run();
assertNull(log.complete(null));
}
@Test
public void testGetTemporaryFilesMultipleFolders() throws IOException {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
Files.createDirectories(dataFolder2.toPath());
SSTableReader[] sstables = {
sstable(dataFolder1, cfs, 0, 128),
sstable(dataFolder1, cfs, 1, 128),
sstable(dataFolder2, cfs, 2, 128),
sstable(dataFolder2, cfs, 3, 128)
};
// they should all have the same number of files since they are created in the same way
int numSStableFiles = sstables[0].getAllFilePaths().size();
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
for (File dataFolder : new File[] {dataFolder1, dataFolder2}) {
Set<File> tmpFiles = getTemporaryFiles(dataFolder);
assertNotNull(tmpFiles);
assertEquals(0, tmpFiles.size());
}
LogTransaction.SSTableTidier[] tidiers = {
log.obsoleted(sstables[0]), log.obsoleted(sstables[2])
};
log.trackNew(sstables[1]);
log.trackNew(sstables[3]);
for (File dataFolder : new File[] {dataFolder1, dataFolder2}) {
Set<File> tmpFiles = getTemporaryFiles(dataFolder);
assertNotNull(tmpFiles);
assertEquals(numSStableFiles, tmpFiles.size());
}
log.finish();
for (File dataFolder : new File[] {dataFolder1, dataFolder2}) {
Set<File> tmpFiles = getTemporaryFiles(dataFolder);
assertNotNull(tmpFiles);
assertEquals(numSStableFiles, tmpFiles.size());
}
sstables[0].markObsolete(tidiers[0]);
sstables[2].markObsolete(tidiers[1]);
Arrays.stream(sstables).forEach(s -> s.selfRef().release());
LogTransaction.waitForDeletions();
for (File dataFolder : new File[] {dataFolder1, dataFolder2}) {
Set<File> tmpFiles = getTemporaryFiles(dataFolder);
assertNotNull(tmpFiles);
assertEquals(0, tmpFiles.size());
}
}
private static void testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
Consumer<LogTransaction> modifier, boolean shouldCommit) throws Throwable {
ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
File dataFolder1 = new File(origiFolder, "1");
File dataFolder2 = new File(origiFolder, "2");
Files.createDirectories(dataFolder1.toPath());
Files.createDirectories(dataFolder2.toPath());
SSTableReader[] sstables = {
sstable(dataFolder1, cfs, 0, 128),
sstable(dataFolder1, cfs, 1, 128),
sstable(dataFolder2, cfs, 2, 128),
sstable(dataFolder2, cfs, 3, 128)
};
LogTransaction log = new LogTransaction(OperationType.COMPACTION);
assertNotNull(log);
LogTransaction.SSTableTidier[] tidiers = {
log.obsoleted(sstables[0]), log.obsoleted(sstables[2])
};
log.trackNew(sstables[1]);
log.trackNew(sstables[3]);
// fake some error condition on the txn logs
modifier.accept(log);
Arrays.stream(sstables).forEach(s -> s.selfRef().release());
LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2));
LogTransaction.waitForDeletions();
if (shouldCommit) {
// only new sstables should still be there
assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[1].getAllFilePaths()));
assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[3].getAllFilePaths()));
} else {
// all files should still be there
assertFiles(
dataFolder1.getPath(),
Sets.newHashSet(
Iterables.concat(
sstables[0].getAllFilePaths(),
sstables[1].getAllFilePaths(),
Collections.singleton(log.logFilePaths().get(0)))));
assertFiles(
dataFolder2.getPath(),
Sets.newHashSet(
Iterables.concat(
sstables[2].getAllFilePaths(),
sstables[3].getAllFilePaths(),
Collections.singleton(log.logFilePaths().get(1)))));
}
// complete the transaction to avoid LEAK errors
Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
log.txnFile().commit(); // just anything to make sure transaction tidier will finish
assertNull(log.complete(null));
}