public static Options parseArgs(String cmdArgs[]) {
CommandLineParser parser = new GnuParser();
CmdLineOptions options = getCmdLineOptions();
try {
CommandLine cmd = parser.parse(options, cmdArgs, false);
if (cmd.hasOption(HELP_OPTION)) {
printUsage(options);
System.exit(0);
}
String[] args = cmd.getArgs();
if (args.length == 0) {
System.err.println("No sstables to split");
printUsage(options);
System.exit(1);
}
Options opts = new Options(Arrays.asList(args));
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.snapshot = !cmd.hasOption(NO_SNAPSHOT_OPTION);
opts.sizeInMB = DEFAULT_SSTABLE_SIZE;
if (cmd.hasOption(SIZE_OPTION))
opts.sizeInMB = Integer.valueOf(cmd.getOptionValue(SIZE_OPTION));
return opts;
} catch (ParseException e) {
errorMsg(e.getMessage(), options);
return null;
}
}
public static Options parseArgs(String cmdArgs[]) {
CommandLineParser parser = new GnuParser();
CmdLineOptions options = getCmdLineOptions();
try {
CommandLine cmd = parser.parse(options, cmdArgs, false);
if (cmd.hasOption(HELP_OPTION)) {
printUsage(options);
System.exit(0);
}
String[] args = cmd.getArgs();
if (args.length >= 4 || args.length < 2) {
String msg = args.length < 2 ? "Missing arguments" : "Too many arguments";
errorMsg(msg, options);
System.exit(1);
}
String keyspace = args[0];
String cf = args[1];
String snapshot = null;
if (args.length == 3) snapshot = args[2];
Options opts = new Options(keyspace, cf, snapshot);
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.keepSource = cmd.hasOption(KEEP_SOURCE);
return opts;
} catch (ParseException e) {
errorMsg(e.getMessage(), options);
return null;
}
}
@Test
public void testRemoveUnfinishedLeftovers_multipleFolders_tooManyFinalRecords() throws Throwable {
testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
txn -> {
List<File> logFiles = txn.logFiles();
Assert.assertEquals(2, logFiles.size());
// insert mismatched records
FileUtils.append(logFiles.get(0), LogRecord.makeCommit(System.currentTimeMillis()).raw);
FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
},
false);
}
@Test
public void testMissingChecksumSecondFromLastLine() throws IOException {
testCorruptRecord(
(t, s) -> { // Fake two lines without a checksum
long now = System.currentTimeMillis();
t.logFiles().forEach(f -> FileUtils.append(f, String.format("add:[ma-3-big,%d,4]", now)));
t.logFiles().forEach(f -> FileUtils.append(f, String.format("commit:[%d,0,0]", now)));
},
false);
}
@Test
public void testMissingChecksumLastLine() throws IOException {
testCorruptRecord(
(t, s) -> {
// Fake a commit without a checksum
long now = System.currentTimeMillis();
t.logFiles().forEach(f -> FileUtils.append(f, String.format("commit:[%d,0,0]", now)));
},
true);
}
@Test
public void testWrongChecksumLastLineWrongRecordFormat() throws IOException {
testCorruptRecord(
(t, s) -> { // Fake a commit with invalid checksum and a wrong record format (extra spaces)
long now = System.currentTimeMillis();
t.logFiles()
.forEach(
f ->
FileUtils.append(f, String.format("commit:[%d ,0 ,0 ][%d]", now, 12345678L)));
},
true);
}
@Test
public void testRemoveUnfinishedLeftovers_multipleFolders_missingFinalRecords_second()
throws Throwable {
testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
txn -> {
List<File> logFiles = txn.logFiles();
Assert.assertEquals(2, logFiles.size());
// insert only one commit record
FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
},
true);
}
@Test
public void testObsoletedDataFileUpdateTimeChanged() throws IOException {
testObsoletedFilesChanged(
sstable -> {
// increase the modification time of the Data file
for (String filePath : sstable.getAllFilePaths()) {
if (filePath.endsWith("Data.db"))
assertTrue(
new File(filePath)
.setLastModified(System.currentTimeMillis() + 60000)); // one minute later
}
});
}
@Test
public void testRemoveUnfinishedLeftovers_multipleFolders_partialFinalRecords_second()
throws Throwable {
testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
txn -> {
List<File> logFiles = txn.logFiles();
Assert.assertEquals(2, logFiles.size());
// insert a full record and a partial one
String finalRecord = LogRecord.makeCommit(System.currentTimeMillis()).raw;
int toChop = finalRecord.length() / 2;
FileUtils.append(logFiles.get(0), finalRecord);
FileUtils.append(
logFiles.get(1), finalRecord.substring(0, finalRecord.length() - toChop));
},
true);
}
@Test
public void testWrongChecksumLastLineMissingFile() throws IOException {
testCorruptRecord(
(t, s) -> { // Fake a commit with invalid checksum and also delete one of the old files
for (String filePath : s.getAllFilePaths()) {
if (filePath.endsWith("Data.db")) {
assertTrue(FileUtils.delete(filePath));
assertNull(t.txnFile().syncFolder(null));
break;
}
}
long now = System.currentTimeMillis();
t.logFiles()
.forEach(
f -> FileUtils.append(f, String.format("commit:[%d,0,0][%d]", now, 12345678L)));
},
false);
}
@Test
public void testRemoveUnfinishedLeftovers_multipleFolders_partialNonFinalRecord_second()
throws Throwable {
testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
txn -> {
List<File> logFiles = txn.logFiles();
Assert.assertEquals(2, logFiles.size());
// insert a partial sstable record and a full commit record
String sstableRecord =
LogRecord.make(LogRecord.Type.ADD, Collections.emptyList(), 0, "abc").raw;
int toChop = sstableRecord.length() / 2;
FileUtils.append(logFiles.get(0), sstableRecord);
FileUtils.append(
logFiles.get(1), sstableRecord.substring(0, sstableRecord.length() - toChop));
String finalRecord = LogRecord.makeCommit(System.currentTimeMillis()).raw;
FileUtils.append(logFiles.get(0), finalRecord);
FileUtils.append(logFiles.get(1), finalRecord);
},
false);
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
String ksName = null;
String cfName = null;
Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
for (String filename : options.filenames) {
File file = new File(filename);
if (!file.exists()) {
System.out.println("Skipping inexisting file " + file);
continue;
}
Pair<Descriptor, Component> pair =
SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null) ksName = desc.ksname;
else if (!ksName.equals(desc.ksname))
throw new IllegalArgumentException("All sstables must be part of the same keyspace");
if (cfName == null) cfName = desc.cfname;
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same column family");
Set<Component> components =
new HashSet<Component>(
Arrays.asList(
new Component[] {
Component.DATA,
Component.PRIMARY_INDEX,
Component.FILTER,
Component.COMPRESSION_INFO,
Component.STATS
}));
Iterator<Component> iter = components.iterator();
while (iter.hasNext()) {
Component component = iter.next();
if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
}
parsedFilenames.put(desc, components);
}
if (ksName == null || cfName == null) {
System.err.println("No valid sstables to split");
System.exit(1);
}
// Do not load sstables since they might be broken
Table table = Table.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
try {
SSTableReader sstable =
SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
sstables.add(sstable);
if (options.snapshot) {
File snapshotDirectory =
Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
sstable.createLinks(snapshotDirectory.getPath());
}
} catch (Exception e) {
System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
if (options.snapshot)
System.out.println(
String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
cfs.getDataTracker().markCompacting(sstables);
for (SSTableReader sstable : sstables) {
try {
new SSTableSplitter(cfs, sstable, options.sizeInMB).split();
// Remove the sstable
sstable.markCompacted();
sstable.releaseReference();
} catch (Exception e) {
System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
private static void errorMsg(String msg, CmdLineOptions options) {
System.err.println(msg);
printUsage(options);
System.exit(1);
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas(false);
if (Schema.instance.getCFMetaData(options.keyspace, options.cf) == null)
throw new IllegalArgumentException(
String.format("Unknown keyspace/columnFamily %s.%s", options.keyspace, options.cf));
Keyspace keyspace = Keyspace.openWithoutSSTables(options.keyspace);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cf);
OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister();
if (options.snapshot != null) lister.onlyBackups(true).snapshots(options.snapshot);
else lister.includeBackups(false);
Collection<SSTableReader> readers = new ArrayList<SSTableReader>();
// Upgrade sstables
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet()) {
Set<Component> components = entry.getValue();
if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX))
continue;
try {
SSTableReader sstable =
SSTableReader.openNoValidation(entry.getKey(), components, cfs.metadata);
if (sstable.descriptor.version.equals(Descriptor.Version.CURRENT)) continue;
readers.add(sstable);
} catch (Exception e) {
JVMStabilityInspector.inspectThrowable(e);
System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
continue;
}
}
int numSSTables = readers.size();
handler.output("Found " + numSSTables + " sstables that need upgrading.");
for (SSTableReader sstable : readers) {
try {
Upgrader upgrader = new Upgrader(cfs, sstable, handler);
upgrader.upgrade();
if (!options.keepSource) {
// Remove the sstable (it's been copied by upgrade)
System.out.format("Deleting table %s.%n", sstable.descriptor.baseFilename());
sstable.markObsolete();
sstable.selfRef().release();
}
} catch (Exception e) {
System.err.println(String.format("Error upgrading %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
SSTableDeletingTask.waitForDeletions();
System.exit(0);
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}