@Test
public void testRemoveUnfinishedLeftovers_multipleFolders_tooManyFinalRecords() throws Throwable {
testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
txn -> {
List<File> logFiles = txn.logFiles();
Assert.assertEquals(2, logFiles.size());
// insert mismatched records
FileUtils.append(logFiles.get(0), LogRecord.makeCommit(System.currentTimeMillis()).raw);
FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
},
false);
}
@Test
public void testMissingChecksumSecondFromLastLine() throws IOException {
testCorruptRecord(
(t, s) -> { // Fake two lines without a checksum
long now = System.currentTimeMillis();
t.logFiles().forEach(f -> FileUtils.append(f, String.format("add:[ma-3-big,%d,4]", now)));
t.logFiles().forEach(f -> FileUtils.append(f, String.format("commit:[%d,0,0]", now)));
},
false);
}
@Test
public void testMissingChecksumLastLine() throws IOException {
testCorruptRecord(
(t, s) -> {
// Fake a commit without a checksum
long now = System.currentTimeMillis();
t.logFiles().forEach(f -> FileUtils.append(f, String.format("commit:[%d,0,0]", now)));
},
true);
}
@Test
public void testWrongChecksumLastLineWrongRecordFormat() throws IOException {
testCorruptRecord(
(t, s) -> { // Fake a commit with invalid checksum and a wrong record format (extra spaces)
long now = System.currentTimeMillis();
t.logFiles()
.forEach(
f ->
FileUtils.append(f, String.format("commit:[%d ,0 ,0 ][%d]", now, 12345678L)));
},
true);
}
@Test
public void testRemoveUnfinishedLeftovers_multipleFolders_missingFinalRecords_second()
throws Throwable {
testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
txn -> {
List<File> logFiles = txn.logFiles();
Assert.assertEquals(2, logFiles.size());
// insert only one commit record
FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
},
true);
}
@Test
public void testObsoletedDataFileUpdateTimeChanged() throws IOException {
testObsoletedFilesChanged(
sstable -> {
// increase the modification time of the Data file
for (String filePath : sstable.getAllFilePaths()) {
if (filePath.endsWith("Data.db"))
assertTrue(
new File(filePath)
.setLastModified(System.currentTimeMillis() + 60000)); // one minute later
}
});
}
@Test
public void testRemoveUnfinishedLeftovers_multipleFolders_partialFinalRecords_second()
throws Throwable {
testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
txn -> {
List<File> logFiles = txn.logFiles();
Assert.assertEquals(2, logFiles.size());
// insert a full record and a partial one
String finalRecord = LogRecord.makeCommit(System.currentTimeMillis()).raw;
int toChop = finalRecord.length() / 2;
FileUtils.append(logFiles.get(0), finalRecord);
FileUtils.append(
logFiles.get(1), finalRecord.substring(0, finalRecord.length() - toChop));
},
true);
}
@Test
public void testWrongChecksumLastLineMissingFile() throws IOException {
testCorruptRecord(
(t, s) -> { // Fake a commit with invalid checksum and also delete one of the old files
for (String filePath : s.getAllFilePaths()) {
if (filePath.endsWith("Data.db")) {
assertTrue(FileUtils.delete(filePath));
assertNull(t.txnFile().syncFolder(null));
break;
}
}
long now = System.currentTimeMillis();
t.logFiles()
.forEach(
f -> FileUtils.append(f, String.format("commit:[%d,0,0][%d]", now, 12345678L)));
},
false);
}
@Test
public void testRemoveUnfinishedLeftovers_multipleFolders_partialNonFinalRecord_second()
throws Throwable {
testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
txn -> {
List<File> logFiles = txn.logFiles();
Assert.assertEquals(2, logFiles.size());
// insert a partial sstable record and a full commit record
String sstableRecord =
LogRecord.make(LogRecord.Type.ADD, Collections.emptyList(), 0, "abc").raw;
int toChop = sstableRecord.length() / 2;
FileUtils.append(logFiles.get(0), sstableRecord);
FileUtils.append(
logFiles.get(1), sstableRecord.substring(0, sstableRecord.length() - toChop));
String finalRecord = LogRecord.makeCommit(System.currentTimeMillis()).raw;
FileUtils.append(logFiles.get(0), finalRecord);
FileUtils.append(logFiles.get(1), finalRecord);
},
false);
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
String ksName = null;
String cfName = null;
Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
for (String filename : options.filenames) {
File file = new File(filename);
if (!file.exists()) {
System.out.println("Skipping inexisting file " + file);
continue;
}
Pair<Descriptor, Component> pair =
SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null) ksName = desc.ksname;
else if (!ksName.equals(desc.ksname))
throw new IllegalArgumentException("All sstables must be part of the same keyspace");
if (cfName == null) cfName = desc.cfname;
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same column family");
Set<Component> components =
new HashSet<Component>(
Arrays.asList(
new Component[] {
Component.DATA,
Component.PRIMARY_INDEX,
Component.FILTER,
Component.COMPRESSION_INFO,
Component.STATS
}));
Iterator<Component> iter = components.iterator();
while (iter.hasNext()) {
Component component = iter.next();
if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
}
parsedFilenames.put(desc, components);
}
if (ksName == null || cfName == null) {
System.err.println("No valid sstables to split");
System.exit(1);
}
// Do not load sstables since they might be broken
Table table = Table.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
try {
SSTableReader sstable =
SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
sstables.add(sstable);
if (options.snapshot) {
File snapshotDirectory =
Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
sstable.createLinks(snapshotDirectory.getPath());
}
} catch (Exception e) {
System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
if (options.snapshot)
System.out.println(
String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
cfs.getDataTracker().markCompacting(sstables);
for (SSTableReader sstable : sstables) {
try {
new SSTableSplitter(cfs, sstable, options.sizeInMB).split();
// Remove the sstable
sstable.markCompacted();
sstable.releaseReference();
} catch (Exception e) {
System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
