public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
String ksName = null;
String cfName = null;
Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
for (String filename : options.filenames) {
File file = new File(filename);
if (!file.exists()) {
System.out.println("Skipping inexisting file " + file);
continue;
}
Pair<Descriptor, Component> pair =
SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null) ksName = desc.ksname;
else if (!ksName.equals(desc.ksname))
throw new IllegalArgumentException("All sstables must be part of the same keyspace");
if (cfName == null) cfName = desc.cfname;
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same column family");
Set<Component> components =
new HashSet<Component>(
Arrays.asList(
new Component[] {
Component.DATA,
Component.PRIMARY_INDEX,
Component.FILTER,
Component.COMPRESSION_INFO,
Component.STATS
}));
Iterator<Component> iter = components.iterator();
while (iter.hasNext()) {
Component component = iter.next();
if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
}
parsedFilenames.put(desc, components);
}
if (ksName == null || cfName == null) {
System.err.println("No valid sstables to split");
System.exit(1);
}
// Do not load sstables since they might be broken
Table table = Table.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
try {
SSTableReader sstable =
SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
sstables.add(sstable);
if (options.snapshot) {
File snapshotDirectory =
Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
sstable.createLinks(snapshotDirectory.getPath());
}
} catch (Exception e) {
System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
if (options.snapshot)
System.out.println(
String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
cfs.getDataTracker().markCompacting(sstables);
for (SSTableReader sstable : sstables) {
try {
new SSTableSplitter(cfs, sstable, options.sizeInMB).split();
// Remove the sstable
sstable.markCompacted();
sstable.releaseReference();
} catch (Exception e) {
System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
@Test
public void testParallelLeveledCompaction() throws Exception {
String ksname = "Keyspace1";
String cfname = "StandardLeveled";
Keyspace keyspace = Keyspace.open(ksname);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(cfname);
store.disableAutoCompaction();
LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) store.getCompactionStrategy();
ByteBuffer value =
ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
// Enough data to have a level 1 and 2
int rows = 128;
int columns = 10;
// Adds enough data to trigger multiple sstable per level
for (int r = 0; r < rows; r++) {
DecoratedKey key = Util.dk(String.valueOf(r));
RowMutation rm = new RowMutation(ksname, key.key);
for (int c = 0; c < columns; c++) {
rm.add(cfname, ByteBufferUtil.bytes("column" + c), value, 0);
}
rm.apply();
store.forceBlockingFlush();
}
// Execute LCS in parallel
ExecutorService executor =
new ThreadPoolExecutor(
4, 4, Long.MAX_VALUE, TimeUnit.SECONDS, new LinkedBlockingDeque<Runnable>());
List<Runnable> tasks = new ArrayList<Runnable>();
while (true) {
while (true) {
final AbstractCompactionTask t = lcs.getMaximalTask(Integer.MIN_VALUE);
if (t == null) break;
tasks.add(
new Runnable() {
public void run() {
t.execute(null);
}
});
}
if (tasks.isEmpty()) break;
List<Future<?>> futures = new ArrayList<Future<?>>(tasks.size());
for (Runnable r : tasks) futures.add(executor.submit(r));
FBUtilities.waitOnFutures(futures);
tasks.clear();
}
// Assert all SSTables are lined up correctly.
LeveledManifest manifest = lcs.manifest;
int levels = manifest.getLevelCount();
for (int level = 0; level < levels; level++) {
List<SSTableReader> sstables = manifest.getLevel(level);
// score check
assert (double) SSTable.getTotalBytes(sstables) / manifest.maxBytesForLevel(level) < 1.00;
// overlap check for levels greater than 0
if (level > 0) {
for (SSTableReader sstable : sstables) {
Set<SSTableReader> overlaps = LeveledManifest.overlapping(sstable, sstables);
assert overlaps.size() == 1 && overlaps.contains(sstable);
}
}
}
for (SSTableReader sstable : store.getSSTables()) {
assert sstable.getSSTableLevel() == sstable.getSSTableLevel();
}
}