private static void configureTransportFactory(
ITransportFactory transportFactory, LoaderOptions opts) {
Map<String, String> options = new HashMap<>();
// If the supplied factory supports the same set of options as our SSL impl, set those
if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE))
options.put(SSLTransportFactory.TRUSTSTORE, opts.encOptions.truststore);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE_PASSWORD))
options.put(SSLTransportFactory.TRUSTSTORE_PASSWORD, opts.encOptions.truststore_password);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.PROTOCOL))
options.put(SSLTransportFactory.PROTOCOL, opts.encOptions.protocol);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.CIPHER_SUITES))
options.put(
SSLTransportFactory.CIPHER_SUITES, Joiner.on(',').join(opts.encOptions.cipher_suites));
if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE)
&& opts.encOptions.require_client_auth)
options.put(SSLTransportFactory.KEYSTORE, opts.encOptions.keystore);
if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE_PASSWORD)
&& opts.encOptions.require_client_auth)
options.put(SSLTransportFactory.KEYSTORE_PASSWORD, opts.encOptions.keystore_password);
// Now check if any of the factory's supported options are set as system properties
for (String optionKey : transportFactory.supportedOptions())
if (System.getProperty(optionKey) != null)
options.put(optionKey, System.getProperty(optionKey));
transportFactory.setOptions(options);
}
public static void main(String args[]) throws IOException {
LoaderOptions options = LoaderOptions.parseArgs(args);
try {
SSTableLoader loader =
new SSTableLoader(options.directory, new ExternalClient(options), options);
SSTableLoader.LoaderFuture future = loader.stream(options.ignores);
if (options.noProgress) {
future.get();
} else {
ProgressIndicator indicator = new ProgressIndicator(future.getPendingFiles());
indicator.start();
System.out.println("");
while (!future.isDone()) {
if (indicator.printProgress()) {
// We're done with streaming
System.out.println("\nWaiting for targets to rebuild indexes ...");
future.get();
assert future.isDone();
} else {
try {
Thread.sleep(1000L);
} catch (Exception e) {
}
}
}
}
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
public static LoaderOptions parseArgs(String cmdArgs[]) {
CommandLineParser parser = new GnuParser();
CmdLineOptions options = getCmdLineOptions();
try {
CommandLine cmd = parser.parse(options, cmdArgs, false);
if (cmd.hasOption(HELP_OPTION)) {
printUsage(options);
System.exit(0);
}
String[] args = cmd.getArgs();
if (args.length == 0) {
System.err.println("Missing sstable directory argument");
printUsage(options);
System.exit(1);
}
if (args.length > 1) {
System.err.println("Too many arguments");
printUsage(options);
System.exit(1);
}
String dirname = args[0];
File dir = new File(dirname);
if (!dir.exists()) errorMsg("Unknown directory: " + dirname, options);
if (!dir.isDirectory()) errorMsg(dirname + " is not a directory", options);
LoaderOptions opts = new LoaderOptions(dir);
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.noProgress = cmd.hasOption(NOPROGRESS_OPTION);
if (cmd.hasOption(IGNORE_NODES_OPTION)) {
String[] nodes = cmd.getOptionValue(IGNORE_NODES_OPTION).split(",");
try {
for (String node : nodes) {
opts.ignores.add(InetAddress.getByName(node));
}
} catch (UnknownHostException e) {
errorMsg(e.getMessage(), options);
}
}
return opts;
} catch (ParseException e) {
errorMsg(e.getMessage(), options);
return null;
}
}
public static void printUsage(Options options) {
String usage = String.format("%s [options] <dir_path>", TOOL_NAME);
String header =
System.lineSeparator()
+ "Bulk load the sstables found in the directory <dir_path> to the configured cluster."
+ "The parent directories of <dir_path> are used as the target keyspace/table name. "
+ "So for instance, to load an sstable named Standard1-g-1-Data.db into Keyspace1/Standard1, "
+ "you will need to have the files Standard1-g-1-Data.db and Standard1-g-1-Index.db into a directory /path/to/Keyspace1/Standard1/.";
String footer =
System.lineSeparator()
+ "You can provide cassandra.yaml file with -f command line option to set up streaming throughput, client and server encryption options. "
+ "Only stream_throughput_outbound_megabits_per_sec, server_encryption_options and client_encryption_options are read from yaml. "
+ "You can override options read from cassandra.yaml with corresponding command line options.";
new HelpFormatter().printHelp(usage, header, options, footer);
}
public static void main(String args[]) {
Config.setClientMode(true);
LoaderOptions options = LoaderOptions.parseArgs(args);
OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
SSTableLoader loader =
new SSTableLoader(
options.directory,
new ExternalClient(
options.hosts,
options.rpcPort,
options.user,
options.passwd,
options.transportFactory,
options.storagePort,
options.sslStoragePort,
options.serverEncOptions),
handler);
DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(options.throttle);
StreamResultFuture future = null;
try {
if (options.noProgress) future = loader.stream(options.ignores);
else future = loader.stream(options.ignores, new ProgressIndicator());
} catch (Exception e) {
System.err.println(e.getMessage());
if (e.getCause() != null) System.err.println(e.getCause());
if (options.debug) e.printStackTrace(System.err);
else System.err.println("Run with --debug to get full stack trace or --help to get help.");
System.exit(1);
}
handler.output(String.format("Streaming session ID: %s", future.planId));
try {
future.get();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println("Streaming to the following hosts failed:");
System.err.println(loader.getFailedHosts());
System.err.println(e);
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
/**
* Insert a record in the database. Any field/value pairs in the specified values HashMap will be
* written into the record with the specified record key.
*
* @param table The name of the table
* @param key The record key of the record to insert.
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
public int insert(String table, String key, HashMap<String, ByteIterator> values) {
if (!_table.equals(table)) {
try {
client.set_keyspace(table);
_table = table;
} catch (Exception e) {
e.printStackTrace();
e.printStackTrace(System.out);
return Error;
}
}
for (int i = 0; i < OperationRetries; i++) {
if (_debug) {
System.out.println("Inserting key: " + key);
}
try {
ByteBuffer wrappedKey = ByteBuffer.wrap(key.getBytes("UTF-8"));
ColumnOrSuperColumn column;
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
column = new ColumnOrSuperColumn();
Column subColumn = new Column(ByteBuffer.wrap(entry.getKey().getBytes("UTF-8")));
subColumn.setValue(ByteBuffer.wrap(entry.getValue().toArray()));
subColumn.setTimestamp(System.currentTimeMillis());
column.setColumn(subColumn);
mutations.add(new Mutation().setColumn_or_supercolumn(column));
}
mutationMap.put(column_family, mutations);
record.put(wrappedKey, mutationMap);
client.batch_mutate(record, ConsistencyLevel.ONE);
mutations.clear();
mutationMap.clear();
record.clear();
return Ok;
} catch (Exception e) {
errorexception = e;
}
try {
Thread.sleep(500);
} catch (InterruptedException e) {
}
}
errorexception.printStackTrace();
errorexception.printStackTrace(System.out);
return Error;
}
public void handleStreamEvent(StreamEvent event) {
if (event.eventType == StreamEvent.Type.STREAM_PREPARED) {
SessionInfo session = ((StreamEvent.SessionPreparedEvent) event).session;
sessionsByHost.put(session.peer, session);
} else if (event.eventType == StreamEvent.Type.FILE_PROGRESS) {
ProgressInfo progressInfo = ((StreamEvent.ProgressEvent) event).progress;
// update progress
Set<ProgressInfo> progresses = progressByHost.get(progressInfo.peer);
if (progresses == null) {
progresses = Sets.newSetFromMap(new ConcurrentHashMap<ProgressInfo, Boolean>());
progressByHost.put(progressInfo.peer, progresses);
}
if (progresses.contains(progressInfo)) progresses.remove(progressInfo);
progresses.add(progressInfo);
StringBuilder sb = new StringBuilder();
sb.append("\rprogress: ");
long totalProgress = 0;
long totalSize = 0;
for (Map.Entry<InetAddress, Set<ProgressInfo>> entry : progressByHost.entrySet()) {
SessionInfo session = sessionsByHost.get(entry.getKey());
long size = session.getTotalSizeToSend();
long current = 0;
int completed = 0;
for (ProgressInfo progress : entry.getValue()) {
if (progress.currentBytes == progress.totalBytes) completed++;
current += progress.currentBytes;
}
totalProgress += current;
totalSize += size;
sb.append("[").append(entry.getKey());
sb.append(" ").append(completed).append("/").append(session.getTotalFilesToSend());
sb.append(" (").append(size == 0 ? 100L : current * 100L / size).append("%)] ");
}
long time = System.nanoTime();
long deltaTime = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(time - lastTime));
lastTime = time;
long deltaProgress = totalProgress - lastProgress;
lastProgress = totalProgress;
sb.append("[total: ")
.append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
.append("% - ");
sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
sb.append(" (avg: ")
.append(mbPerSec(totalProgress, TimeUnit.NANOSECONDS.toMillis(time - start)))
.append("MB/s)]");
System.out.print(sb.toString());
}
}
/** test get_count() to work correctly with 'count' settings around page size. (CASSANDRA-4833) */
@Test
public void test_get_count() throws Exception {
Schema.instance.clear(); // Schema are now written on disk and will be reloaded
new EmbeddedCassandraService().start();
ThriftSessionManager.instance.setCurrentSocket(new InetSocketAddress(9160));
DecoratedKey key = Util.dk("testkey");
for (int i = 0; i < 3050; i++) {
RowMutation rm = new RowMutation("Keyspace1", key.key);
rm.add(
new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i))),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.apply();
}
CassandraServer server = new CassandraServer();
server.set_keyspace("Keyspace1");
// same as page size
int count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(1024), ConsistencyLevel.ONE);
assert count == 1024 : "expected 1024 but was " + count;
// 1 above page size
count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(1025), ConsistencyLevel.ONE);
assert count == 1025 : "expected 1025 but was " + count;
// above number of columns
count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(4000), ConsistencyLevel.ONE);
assert count == 3050 : "expected 3050 but was " + count;
// same as number of columns
count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(3050), ConsistencyLevel.ONE);
assert count == 3050 : "expected 3050 but was " + count;
// 1 above number of columns
count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(3051), ConsistencyLevel.ONE);
assert count == 3050 : "expected 3050 but was " + count;
}
public boolean printProgress() {
boolean done = true;
StringBuilder sb = new StringBuilder();
sb.append("\rprogress: ");
long totalProgress = 0;
long totalSize = 0;
for (Map.Entry<InetAddress, Collection<PendingFile>> entry : filesByHost.entrySet()) {
long progress = 0;
long size = 0;
int completed = 0;
Collection<PendingFile> pendings = entry.getValue();
for (PendingFile f : pendings) {
progress += f.progress;
size += f.size;
if (f.progress == f.size) completed++;
}
totalProgress += progress;
totalSize += size;
if (completed != pendings.size()) done = false;
sb.append("[").append(entry.getKey());
sb.append(" ").append(completed).append("/").append(pendings.size());
sb.append(" (").append(size == 0 ? 100L : progress * 100L / size).append(")] ");
}
long time = System.currentTimeMillis();
long deltaTime = time - lastTime;
lastTime = time;
long deltaProgress = totalProgress - lastProgress;
lastProgress = totalProgress;
sb.append("[total: ")
.append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
.append(" - ");
sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
sb.append(" (avg: ").append(mbPerSec(totalProgress, time - startTime)).append("MB/s)]");
;
System.out.print(sb.toString());
return done;
}
/**
* Delete a record from the database.
*
* @param table The name of the table
* @param key The record key of the record to delete.
* @return Zero on success, a non-zero error code on error
*/
public int delete(String table, String key) {
if (!_table.equals(table)) {
try {
client.set_keyspace(table);
_table = table;
} catch (Exception e) {
e.printStackTrace();
e.printStackTrace(System.out);
return Error;
}
}
for (int i = 0; i < OperationRetries; i++) {
try {
client.remove(
ByteBuffer.wrap(key.getBytes("UTF-8")),
new ColumnPath(column_family),
System.currentTimeMillis(),
ConsistencyLevel.ONE);
if (_debug) {
System.out.println("Delete key: " + key);
}
return Ok;
} catch (Exception e) {
errorexception = e;
}
try {
Thread.sleep(500);
} catch (InterruptedException e) {
}
}
errorexception.printStackTrace();
errorexception.printStackTrace(System.out);
return Error;
}
public static void main(String[] args) {
CassandraClient7 cli = new CassandraClient7();
Properties props = new Properties();
props.setProperty("hosts", args[0]);
cli.setProperties(props);
try {
cli.init();
} catch (Exception e) {
e.printStackTrace();
System.exit(0);
}
HashMap<String, ByteIterator> vals = new HashMap<String, ByteIterator>();
vals.put("age", new StringByteIterator("57"));
vals.put("middlename", new StringByteIterator("bradley"));
vals.put("favoritecolor", new StringByteIterator("blue"));
int res = cli.insert("usertable", "BrianFrankCooper", vals);
System.out.println("Result of insert: " + res);
HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
HashSet<String> fields = new HashSet<String>();
fields.add("middlename");
fields.add("age");
fields.add("favoritecolor");
res = cli.read("usertable", "BrianFrankCooper", null, result);
System.out.println("Result of read: " + res);
for (String s : result.keySet()) {
System.out.println("[" + s + "]=[" + result.get(s) + "]");
}
res = cli.delete("usertable", "BrianFrankCooper");
System.out.println("Result of delete: " + res);
}
public void start() {
startTime = System.currentTimeMillis();
}
private static void errorMsg(String msg, CmdLineOptions options) {
System.err.println(msg);
printUsage(options);
System.exit(1);
}
public static LoaderOptions parseArgs(String cmdArgs[]) {
CommandLineParser parser = new GnuParser();
CmdLineOptions options = getCmdLineOptions();
try {
CommandLine cmd = parser.parse(options, cmdArgs, false);
if (cmd.hasOption(HELP_OPTION)) {
printUsage(options);
System.exit(0);
}
String[] args = cmd.getArgs();
if (args.length == 0) {
System.err.println("Missing sstable directory argument");
printUsage(options);
System.exit(1);
}
if (args.length > 1) {
System.err.println("Too many arguments");
printUsage(options);
System.exit(1);
}
String dirname = args[0];
File dir = new File(dirname);
if (!dir.exists()) errorMsg("Unknown directory: " + dirname, options);
if (!dir.isDirectory()) errorMsg(dirname + " is not a directory", options);
LoaderOptions opts = new LoaderOptions(dir);
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.noProgress = cmd.hasOption(NOPROGRESS_OPTION);
if (cmd.hasOption(RPC_PORT_OPTION))
opts.rpcPort = Integer.parseInt(cmd.getOptionValue(RPC_PORT_OPTION));
if (cmd.hasOption(USER_OPTION)) opts.user = cmd.getOptionValue(USER_OPTION);
if (cmd.hasOption(PASSWD_OPTION)) opts.passwd = cmd.getOptionValue(PASSWD_OPTION);
if (cmd.hasOption(INITIAL_HOST_ADDRESS_OPTION)) {
String[] nodes = cmd.getOptionValue(INITIAL_HOST_ADDRESS_OPTION).split(",");
try {
for (String node : nodes) {
opts.hosts.add(InetAddress.getByName(node.trim()));
}
} catch (UnknownHostException e) {
errorMsg("Unknown host: " + e.getMessage(), options);
}
} else {
System.err.println("Initial hosts must be specified (-d)");
printUsage(options);
System.exit(1);
}
if (cmd.hasOption(IGNORE_NODES_OPTION)) {
String[] nodes = cmd.getOptionValue(IGNORE_NODES_OPTION).split(",");
try {
for (String node : nodes) {
opts.ignores.add(InetAddress.getByName(node.trim()));
}
} catch (UnknownHostException e) {
errorMsg("Unknown host: " + e.getMessage(), options);
}
}
// try to load config file first, so that values can be rewritten with other option values.
// otherwise use default config.
Config config;
if (cmd.hasOption(CONFIG_PATH)) {
File configFile = new File(cmd.getOptionValue(CONFIG_PATH));
if (!configFile.exists()) {
errorMsg("Config file not found", options);
}
config = new YamlConfigurationLoader().loadConfig(configFile.toURI().toURL());
} else {
config = new Config();
}
opts.storagePort = config.storage_port;
opts.sslStoragePort = config.ssl_storage_port;
opts.throttle = config.stream_throughput_outbound_megabits_per_sec;
opts.encOptions = config.client_encryption_options;
opts.serverEncOptions = config.server_encryption_options;
if (cmd.hasOption(THROTTLE_MBITS)) {
opts.throttle = Integer.parseInt(cmd.getOptionValue(THROTTLE_MBITS));
}
if (cmd.hasOption(SSL_TRUSTSTORE)) {
opts.encOptions.truststore = cmd.getOptionValue(SSL_TRUSTSTORE);
}
if (cmd.hasOption(SSL_TRUSTSTORE_PW)) {
opts.encOptions.truststore_password = cmd.getOptionValue(SSL_TRUSTSTORE_PW);
}
if (cmd.hasOption(SSL_KEYSTORE)) {
opts.encOptions.keystore = cmd.getOptionValue(SSL_KEYSTORE);
// if a keystore was provided, lets assume we'll need to use it
opts.encOptions.require_client_auth = true;
}
if (cmd.hasOption(SSL_KEYSTORE_PW)) {
opts.encOptions.keystore_password = cmd.getOptionValue(SSL_KEYSTORE_PW);
}
if (cmd.hasOption(SSL_PROTOCOL)) {
opts.encOptions.protocol = cmd.getOptionValue(SSL_PROTOCOL);
}
if (cmd.hasOption(SSL_ALGORITHM)) {
opts.encOptions.algorithm = cmd.getOptionValue(SSL_ALGORITHM);
}
if (cmd.hasOption(SSL_STORE_TYPE)) {
opts.encOptions.store_type = cmd.getOptionValue(SSL_STORE_TYPE);
}
if (cmd.hasOption(SSL_CIPHER_SUITES)) {
opts.encOptions.cipher_suites = cmd.getOptionValue(SSL_CIPHER_SUITES).split(",");
}
if (cmd.hasOption(TRANSPORT_FACTORY)) {
ITransportFactory transportFactory =
getTransportFactory(cmd.getOptionValue(TRANSPORT_FACTORY));
configureTransportFactory(transportFactory, opts);
opts.transportFactory = transportFactory;
}
return opts;
} catch (ParseException | ConfigurationException | MalformedURLException e) {
errorMsg(e.getMessage(), options);
return null;
}
}
public ProgressIndicator() {
start = lastTime = System.nanoTime();
}