public void run() {
int baseKey = (index * keyCount) % keys.size();
for (int i = 0; i < requests; i++) {
int keyIndex = (baseKey + (i % keyCount)) % keys.size();
String key = keys.get(keyIndex);
Transaction txn = fac.create();
long start = System.currentTimeMillis();
try {
txn.begin();
byte[] value = txn.read(key);
txn.commit();
timeElapsed += System.currentTimeMillis() - start;
if (!silent.get()) {
System.out.println("[Thread-" + index + "] " + key + ": " + new String(value));
}
success++;
} catch (Exception e) {
timeElapsed += System.currentTimeMillis() - start;
if (!silent.get()) {
System.out.println("[Thread-" + index + "] Error: " + e.getMessage());
}
failure++;
}
}
}
public static void main(String args[]) throws IOException {
LoaderOptions options = LoaderOptions.parseArgs(args);
try {
SSTableLoader loader =
new SSTableLoader(options.directory, new ExternalClient(options), options);
SSTableLoader.LoaderFuture future = loader.stream(options.ignores);
if (options.noProgress) {
future.get();
} else {
ProgressIndicator indicator = new ProgressIndicator(future.getPendingFiles());
indicator.start();
System.out.println("");
while (!future.isDone()) {
if (indicator.printProgress()) {
// We're done with streaming
System.out.println("\nWaiting for targets to rebuild indexes ...");
future.get();
assert future.isDone();
} else {
try {
Thread.sleep(1000L);
} catch (Exception e) {
}
}
}
}
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
public static void toLibSVM(List<TrainingSample<BxZoneLabel>> trainingElements, String filePath)
throws IOException {
BufferedWriter svmDataFile = null;
try {
FileWriter fstream = new FileWriter(filePath);
svmDataFile = new BufferedWriter(fstream);
for (TrainingSample<BxZoneLabel> elem : trainingElements) {
if (elem.getLabel() == null) {
continue;
}
svmDataFile.write(String.valueOf(elem.getLabel().ordinal()));
svmDataFile.write(" ");
Integer featureCounter = 1;
for (Double value : elem.getFeatureVector().getValues()) {
StringBuilder sb = new StringBuilder();
Formatter formatter = new Formatter(sb, Locale.US);
formatter.format("%d:%.5f", featureCounter++, value);
svmDataFile.write(sb.toString());
svmDataFile.write(" ");
}
svmDataFile.write("\n");
}
svmDataFile.close();
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
return;
} finally {
if (svmDataFile != null) {
svmDataFile.close();
}
}
System.out.println("Done.");
}
// Take a tree of files starting in a directory in a zip file
// and copy them to a disk directory, recreating the tree.
private int unpackZipFile(
File inZipFile, String directory, String parent, boolean suppressFirstPathElement) {
int count = 0;
if (!inZipFile.exists()) return count;
parent = parent.trim();
if (!parent.endsWith(File.separator)) parent += File.separator;
if (!directory.endsWith(File.separator)) directory += File.separator;
File outFile = null;
try {
ZipFile zipFile = new ZipFile(inZipFile);
Enumeration zipEntries = zipFile.entries();
while (zipEntries.hasMoreElements()) {
ZipEntry entry = (ZipEntry) zipEntries.nextElement();
String name = entry.getName().replace('/', File.separatorChar);
if (name.startsWith(directory)) {
if (suppressFirstPathElement) name = name.substring(directory.length());
outFile = new File(parent + name);
// Create the directory, just in case
if (name.indexOf(File.separatorChar) >= 0) {
String p = name.substring(0, name.lastIndexOf(File.separatorChar) + 1);
File dirFile = new File(parent + p);
dirFile.mkdirs();
}
if (!entry.isDirectory()) {
System.out.println("Installing " + outFile);
// Copy the file
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outFile));
BufferedInputStream in = new BufferedInputStream(zipFile.getInputStream(entry));
int size = 1024;
int n = 0;
byte[] b = new byte[size];
while ((n = in.read(b, 0, size)) != -1) out.write(b, 0, n);
in.close();
out.flush();
out.close();
// Count the file
count++;
}
}
}
zipFile.close();
} catch (Exception e) {
System.err.println("...an error occured while installing " + outFile);
e.printStackTrace();
System.err.println("Error copying " + outFile.getName() + "\n" + e.getMessage());
return -count;
}
System.out.println(count + " files were installed.");
return count;
}
private boolean startLauncher(File dir) {
try {
Runtime rt = Runtime.getRuntime();
ArrayList<String> command = new ArrayList<String>();
command.add("java");
command.add("-jar");
command.add("Launcher.jar");
String[] cmdarray = command.toArray(new String[command.size()]);
Process proc = rt.exec(cmdarray, null, dir);
return true;
} catch (Exception ex) {
System.err.println("Unable to start the Launcher program.\n" + ex.getMessage());
return false;
}
}
public static void main(String args[]) {
Config.setClientMode(true);
LoaderOptions options = LoaderOptions.parseArgs(args);
OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
SSTableLoader loader =
new SSTableLoader(
options.directory,
new ExternalClient(
options.hosts,
options.rpcPort,
options.user,
options.passwd,
options.transportFactory,
options.storagePort,
options.sslStoragePort,
options.serverEncOptions),
handler);
DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(options.throttle);
StreamResultFuture future = null;
try {
if (options.noProgress) future = loader.stream(options.ignores);
else future = loader.stream(options.ignores, new ProgressIndicator());
} catch (Exception e) {
System.err.println(e.getMessage());
if (e.getCause() != null) System.err.println(e.getCause());
if (options.debug) e.printStackTrace(System.err);
else System.err.println("Run with --debug to get full stack trace or --help to get help.");
System.exit(1);
}
handler.output(String.format("Streaming session ID: %s", future.planId));
try {
future.get();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println("Streaming to the following hosts failed:");
System.err.println(loader.getFailedHosts());
System.err.println(e);
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
String ksName = null;
String cfName = null;
Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
for (String filename : options.filenames) {
File file = new File(filename);
if (!file.exists()) {
System.out.println("Skipping inexisting file " + file);
continue;
}
Pair<Descriptor, Component> pair =
SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null) ksName = desc.ksname;
else if (!ksName.equals(desc.ksname))
throw new IllegalArgumentException("All sstables must be part of the same keyspace");
if (cfName == null) cfName = desc.cfname;
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same column family");
Set<Component> components =
new HashSet<Component>(
Arrays.asList(
new Component[] {
Component.DATA,
Component.PRIMARY_INDEX,
Component.FILTER,
Component.COMPRESSION_INFO,
Component.STATS
}));
Iterator<Component> iter = components.iterator();
while (iter.hasNext()) {
Component component = iter.next();
if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
}
parsedFilenames.put(desc, components);
}
if (ksName == null || cfName == null) {
System.err.println("No valid sstables to split");
System.exit(1);
}
// Do not load sstables since they might be broken
Table table = Table.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
try {
SSTableReader sstable =
SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
sstables.add(sstable);
if (options.snapshot) {
File snapshotDirectory =
Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
sstable.createLinks(snapshotDirectory.getPath());
}
} catch (Exception e) {
System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
if (options.snapshot)
System.out.println(
String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
cfs.getDataTracker().markCompacting(sstables);
for (SSTableReader sstable : sstables) {
try {
new SSTableSplitter(cfs, sstable, options.sizeInMB).split();
// Remove the sstable
sstable.markCompacted();
sstable.releaseReference();
} catch (Exception e) {
System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas(false);
if (Schema.instance.getCFMetaData(options.keyspace, options.cf) == null)
throw new IllegalArgumentException(
String.format("Unknown keyspace/columnFamily %s.%s", options.keyspace, options.cf));
Keyspace keyspace = Keyspace.openWithoutSSTables(options.keyspace);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cf);
OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister();
if (options.snapshot != null) lister.onlyBackups(true).snapshots(options.snapshot);
else lister.includeBackups(false);
Collection<SSTableReader> readers = new ArrayList<SSTableReader>();
// Upgrade sstables
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet()) {
Set<Component> components = entry.getValue();
if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX))
continue;
try {
SSTableReader sstable =
SSTableReader.openNoValidation(entry.getKey(), components, cfs.metadata);
if (sstable.descriptor.version.equals(Descriptor.Version.CURRENT)) continue;
readers.add(sstable);
} catch (Exception e) {
JVMStabilityInspector.inspectThrowable(e);
System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
continue;
}
}
int numSSTables = readers.size();
handler.output("Found " + numSSTables + " sstables that need upgrading.");
for (SSTableReader sstable : readers) {
try {
Upgrader upgrader = new Upgrader(cfs, sstable, handler);
upgrader.upgrade();
if (!options.keepSource) {
// Remove the sstable (it's been copied by upgrade)
System.out.format("Deleting table %s.%n", sstable.descriptor.baseFilename());
sstable.markObsolete();
sstable.selfRef().release();
}
} catch (Exception e) {
System.err.println(String.format("Error upgrading %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
SSTableDeletingTask.waitForDeletions();
System.exit(0);
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
