private static void configureTransportFactory(
        ITransportFactory transportFactory, LoaderOptions opts) {
      Map<String, String> options = new HashMap<>();
      // If the supplied factory supports the same set of options as our SSL impl, set those
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE))
        options.put(SSLTransportFactory.TRUSTSTORE, opts.encOptions.truststore);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE_PASSWORD))
        options.put(SSLTransportFactory.TRUSTSTORE_PASSWORD, opts.encOptions.truststore_password);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.PROTOCOL))
        options.put(SSLTransportFactory.PROTOCOL, opts.encOptions.protocol);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.CIPHER_SUITES))
        options.put(
            SSLTransportFactory.CIPHER_SUITES, Joiner.on(',').join(opts.encOptions.cipher_suites));

      if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE)
          && opts.encOptions.require_client_auth)
        options.put(SSLTransportFactory.KEYSTORE, opts.encOptions.keystore);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE_PASSWORD)
          && opts.encOptions.require_client_auth)
        options.put(SSLTransportFactory.KEYSTORE_PASSWORD, opts.encOptions.keystore_password);

      // Now check if any of the factory's supported options are set as system properties
      for (String optionKey : transportFactory.supportedOptions())
        if (System.getProperty(optionKey) != null)
          options.put(optionKey, System.getProperty(optionKey));

      transportFactory.setOptions(options);
    }
  public static void main(String args[]) throws IOException {
    LoaderOptions options = LoaderOptions.parseArgs(args);
    try {
      SSTableLoader loader =
          new SSTableLoader(options.directory, new ExternalClient(options), options);
      SSTableLoader.LoaderFuture future = loader.stream(options.ignores);

      if (options.noProgress) {
        future.get();
      } else {
        ProgressIndicator indicator = new ProgressIndicator(future.getPendingFiles());
        indicator.start();
        System.out.println("");
        while (!future.isDone()) {
          if (indicator.printProgress()) {
            // We're done with streaming
            System.out.println("\nWaiting for targets to rebuild indexes ...");
            future.get();
            assert future.isDone();
          } else {
            try {
              Thread.sleep(1000L);
            } catch (Exception e) {
            }
          }
        }
      }

      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }
    public static LoaderOptions parseArgs(String cmdArgs[]) {
      CommandLineParser parser = new GnuParser();
      CmdLineOptions options = getCmdLineOptions();
      try {
        CommandLine cmd = parser.parse(options, cmdArgs, false);

        if (cmd.hasOption(HELP_OPTION)) {
          printUsage(options);
          System.exit(0);
        }

        String[] args = cmd.getArgs();
        if (args.length == 0) {
          System.err.println("Missing sstable directory argument");
          printUsage(options);
          System.exit(1);
        }

        if (args.length > 1) {
          System.err.println("Too many arguments");
          printUsage(options);
          System.exit(1);
        }

        String dirname = args[0];
        File dir = new File(dirname);

        if (!dir.exists()) errorMsg("Unknown directory: " + dirname, options);

        if (!dir.isDirectory()) errorMsg(dirname + " is not a directory", options);

        LoaderOptions opts = new LoaderOptions(dir);

        opts.debug = cmd.hasOption(DEBUG_OPTION);
        opts.verbose = cmd.hasOption(VERBOSE_OPTION);
        opts.noProgress = cmd.hasOption(NOPROGRESS_OPTION);

        if (cmd.hasOption(IGNORE_NODES_OPTION)) {
          String[] nodes = cmd.getOptionValue(IGNORE_NODES_OPTION).split(",");
          try {
            for (String node : nodes) {
              opts.ignores.add(InetAddress.getByName(node));
            }
          } catch (UnknownHostException e) {
            errorMsg(e.getMessage(), options);
          }
        }

        return opts;
      } catch (ParseException e) {
        errorMsg(e.getMessage(), options);
        return null;
      }
    }
 public static void printUsage(Options options) {
   String usage = String.format("%s [options] <dir_path>", TOOL_NAME);
   String header =
       System.lineSeparator()
           + "Bulk load the sstables found in the directory <dir_path> to the configured cluster."
           + "The parent directories of <dir_path> are used as the target keyspace/table name. "
           + "So for instance, to load an sstable named Standard1-g-1-Data.db into Keyspace1/Standard1, "
           + "you will need to have the files Standard1-g-1-Data.db and Standard1-g-1-Index.db into a directory /path/to/Keyspace1/Standard1/.";
   String footer =
       System.lineSeparator()
           + "You can provide cassandra.yaml file with -f command line option to set up streaming throughput, client and server encryption options. "
           + "Only stream_throughput_outbound_megabits_per_sec, server_encryption_options and client_encryption_options are read from yaml. "
           + "You can override options read from cassandra.yaml with corresponding command line options.";
   new HelpFormatter().printHelp(usage, header, options, footer);
 }
  public static void main(String args[]) {
    Config.setClientMode(true);
    LoaderOptions options = LoaderOptions.parseArgs(args);
    OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
    SSTableLoader loader =
        new SSTableLoader(
            options.directory,
            new ExternalClient(
                options.hosts,
                options.rpcPort,
                options.user,
                options.passwd,
                options.transportFactory,
                options.storagePort,
                options.sslStoragePort,
                options.serverEncOptions),
            handler);
    DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(options.throttle);
    StreamResultFuture future = null;
    try {
      if (options.noProgress) future = loader.stream(options.ignores);
      else future = loader.stream(options.ignores, new ProgressIndicator());
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (e.getCause() != null) System.err.println(e.getCause());
      if (options.debug) e.printStackTrace(System.err);
      else System.err.println("Run with --debug to get full stack trace or --help to get help.");
      System.exit(1);
    }

    handler.output(String.format("Streaming session ID: %s", future.planId));

    try {
      future.get();
      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println("Streaming to the following hosts failed:");
      System.err.println(loader.getFailedHosts());
      System.err.println(e);
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }
Пример #6
0
  /**
   * Insert a record in the database. Any field/value pairs in the specified values HashMap will be
   * written into the record with the specified record key.
   *
   * @param table The name of the table
   * @param key The record key of the record to insert.
   * @param values A HashMap of field/value pairs to insert in the record
   * @return Zero on success, a non-zero error code on error
   */
  public int insert(String table, String key, HashMap<String, ByteIterator> values) {
    if (!_table.equals(table)) {
      try {
        client.set_keyspace(table);
        _table = table;
      } catch (Exception e) {
        e.printStackTrace();
        e.printStackTrace(System.out);
        return Error;
      }
    }

    for (int i = 0; i < OperationRetries; i++) {
      if (_debug) {
        System.out.println("Inserting key: " + key);
      }

      try {
        ByteBuffer wrappedKey = ByteBuffer.wrap(key.getBytes("UTF-8"));

        ColumnOrSuperColumn column;
        for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
          column = new ColumnOrSuperColumn();
          Column subColumn = new Column(ByteBuffer.wrap(entry.getKey().getBytes("UTF-8")));
          subColumn.setValue(ByteBuffer.wrap(entry.getValue().toArray()));
          subColumn.setTimestamp(System.currentTimeMillis());
          column.setColumn(subColumn);

          mutations.add(new Mutation().setColumn_or_supercolumn(column));
        }

        mutationMap.put(column_family, mutations);
        record.put(wrappedKey, mutationMap);

        client.batch_mutate(record, ConsistencyLevel.ONE);

        mutations.clear();
        mutationMap.clear();
        record.clear();

        return Ok;
      } catch (Exception e) {
        errorexception = e;
      }
      try {
        Thread.sleep(500);
      } catch (InterruptedException e) {
      }
    }

    errorexception.printStackTrace();
    errorexception.printStackTrace(System.out);
    return Error;
  }
    public void handleStreamEvent(StreamEvent event) {
      if (event.eventType == StreamEvent.Type.STREAM_PREPARED) {
        SessionInfo session = ((StreamEvent.SessionPreparedEvent) event).session;
        sessionsByHost.put(session.peer, session);
      } else if (event.eventType == StreamEvent.Type.FILE_PROGRESS) {
        ProgressInfo progressInfo = ((StreamEvent.ProgressEvent) event).progress;

        // update progress
        Set<ProgressInfo> progresses = progressByHost.get(progressInfo.peer);
        if (progresses == null) {
          progresses = Sets.newSetFromMap(new ConcurrentHashMap<ProgressInfo, Boolean>());
          progressByHost.put(progressInfo.peer, progresses);
        }
        if (progresses.contains(progressInfo)) progresses.remove(progressInfo);
        progresses.add(progressInfo);

        StringBuilder sb = new StringBuilder();
        sb.append("\rprogress: ");

        long totalProgress = 0;
        long totalSize = 0;
        for (Map.Entry<InetAddress, Set<ProgressInfo>> entry : progressByHost.entrySet()) {
          SessionInfo session = sessionsByHost.get(entry.getKey());

          long size = session.getTotalSizeToSend();
          long current = 0;
          int completed = 0;
          for (ProgressInfo progress : entry.getValue()) {
            if (progress.currentBytes == progress.totalBytes) completed++;
            current += progress.currentBytes;
          }
          totalProgress += current;
          totalSize += size;
          sb.append("[").append(entry.getKey());
          sb.append(" ").append(completed).append("/").append(session.getTotalFilesToSend());
          sb.append(" (").append(size == 0 ? 100L : current * 100L / size).append("%)] ");
        }
        long time = System.nanoTime();
        long deltaTime = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(time - lastTime));
        lastTime = time;
        long deltaProgress = totalProgress - lastProgress;
        lastProgress = totalProgress;

        sb.append("[total: ")
            .append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
            .append("% - ");
        sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
        sb.append(" (avg: ")
            .append(mbPerSec(totalProgress, TimeUnit.NANOSECONDS.toMillis(time - start)))
            .append("MB/s)]");

        System.out.print(sb.toString());
      }
    }
  /** test get_count() to work correctly with 'count' settings around page size. (CASSANDRA-4833) */
  @Test
  public void test_get_count() throws Exception {
    Schema.instance.clear(); // Schema are now written on disk and will be reloaded
    new EmbeddedCassandraService().start();
    ThriftSessionManager.instance.setCurrentSocket(new InetSocketAddress(9160));

    DecoratedKey key = Util.dk("testkey");
    for (int i = 0; i < 3050; i++) {
      RowMutation rm = new RowMutation("Keyspace1", key.key);
      rm.add(
          new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i))),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          System.currentTimeMillis());
      rm.apply();
    }

    CassandraServer server = new CassandraServer();
    server.set_keyspace("Keyspace1");

    // same as page size
    int count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(1024), ConsistencyLevel.ONE);
    assert count == 1024 : "expected 1024 but was " + count;

    // 1 above page size
    count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(1025), ConsistencyLevel.ONE);
    assert count == 1025 : "expected 1025 but was " + count;

    // above number of columns
    count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(4000), ConsistencyLevel.ONE);
    assert count == 3050 : "expected 3050 but was " + count;

    // same as number of columns
    count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(3050), ConsistencyLevel.ONE);
    assert count == 3050 : "expected 3050 but was " + count;

    // 1 above number of columns
    count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(3051), ConsistencyLevel.ONE);
    assert count == 3050 : "expected 3050 but was " + count;
  }
    public boolean printProgress() {
      boolean done = true;
      StringBuilder sb = new StringBuilder();
      sb.append("\rprogress: ");
      long totalProgress = 0;
      long totalSize = 0;
      for (Map.Entry<InetAddress, Collection<PendingFile>> entry : filesByHost.entrySet()) {
        long progress = 0;
        long size = 0;
        int completed = 0;
        Collection<PendingFile> pendings = entry.getValue();
        for (PendingFile f : pendings) {
          progress += f.progress;
          size += f.size;
          if (f.progress == f.size) completed++;
        }
        totalProgress += progress;
        totalSize += size;
        if (completed != pendings.size()) done = false;
        sb.append("[").append(entry.getKey());
        sb.append(" ").append(completed).append("/").append(pendings.size());
        sb.append(" (").append(size == 0 ? 100L : progress * 100L / size).append(")] ");
      }
      long time = System.currentTimeMillis();
      long deltaTime = time - lastTime;
      lastTime = time;
      long deltaProgress = totalProgress - lastProgress;
      lastProgress = totalProgress;

      sb.append("[total: ")
          .append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
          .append(" - ");
      sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
      sb.append(" (avg: ").append(mbPerSec(totalProgress, time - startTime)).append("MB/s)]");
      ;
      System.out.print(sb.toString());
      return done;
    }
Пример #10
0
  /**
   * Delete a record from the database.
   *
   * @param table The name of the table
   * @param key The record key of the record to delete.
   * @return Zero on success, a non-zero error code on error
   */
  public int delete(String table, String key) {
    if (!_table.equals(table)) {
      try {
        client.set_keyspace(table);
        _table = table;
      } catch (Exception e) {
        e.printStackTrace();
        e.printStackTrace(System.out);
        return Error;
      }
    }

    for (int i = 0; i < OperationRetries; i++) {
      try {
        client.remove(
            ByteBuffer.wrap(key.getBytes("UTF-8")),
            new ColumnPath(column_family),
            System.currentTimeMillis(),
            ConsistencyLevel.ONE);

        if (_debug) {
          System.out.println("Delete key: " + key);
        }

        return Ok;
      } catch (Exception e) {
        errorexception = e;
      }
      try {
        Thread.sleep(500);
      } catch (InterruptedException e) {
      }
    }
    errorexception.printStackTrace();
    errorexception.printStackTrace(System.out);
    return Error;
  }
Пример #11
0
  public static void main(String[] args) {
    CassandraClient7 cli = new CassandraClient7();

    Properties props = new Properties();

    props.setProperty("hosts", args[0]);
    cli.setProperties(props);

    try {
      cli.init();
    } catch (Exception e) {
      e.printStackTrace();
      System.exit(0);
    }

    HashMap<String, ByteIterator> vals = new HashMap<String, ByteIterator>();
    vals.put("age", new StringByteIterator("57"));
    vals.put("middlename", new StringByteIterator("bradley"));
    vals.put("favoritecolor", new StringByteIterator("blue"));
    int res = cli.insert("usertable", "BrianFrankCooper", vals);
    System.out.println("Result of insert: " + res);

    HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
    HashSet<String> fields = new HashSet<String>();
    fields.add("middlename");
    fields.add("age");
    fields.add("favoritecolor");
    res = cli.read("usertable", "BrianFrankCooper", null, result);
    System.out.println("Result of read: " + res);
    for (String s : result.keySet()) {
      System.out.println("[" + s + "]=[" + result.get(s) + "]");
    }

    res = cli.delete("usertable", "BrianFrankCooper");
    System.out.println("Result of delete: " + res);
  }
 public void start() {
   startTime = System.currentTimeMillis();
 }
 private static void errorMsg(String msg, CmdLineOptions options) {
   System.err.println(msg);
   printUsage(options);
   System.exit(1);
 }
Пример #14
0
    public static LoaderOptions parseArgs(String cmdArgs[]) {
      CommandLineParser parser = new GnuParser();
      CmdLineOptions options = getCmdLineOptions();
      try {
        CommandLine cmd = parser.parse(options, cmdArgs, false);

        if (cmd.hasOption(HELP_OPTION)) {
          printUsage(options);
          System.exit(0);
        }

        String[] args = cmd.getArgs();
        if (args.length == 0) {
          System.err.println("Missing sstable directory argument");
          printUsage(options);
          System.exit(1);
        }

        if (args.length > 1) {
          System.err.println("Too many arguments");
          printUsage(options);
          System.exit(1);
        }

        String dirname = args[0];
        File dir = new File(dirname);

        if (!dir.exists()) errorMsg("Unknown directory: " + dirname, options);

        if (!dir.isDirectory()) errorMsg(dirname + " is not a directory", options);

        LoaderOptions opts = new LoaderOptions(dir);

        opts.debug = cmd.hasOption(DEBUG_OPTION);
        opts.verbose = cmd.hasOption(VERBOSE_OPTION);
        opts.noProgress = cmd.hasOption(NOPROGRESS_OPTION);

        if (cmd.hasOption(RPC_PORT_OPTION))
          opts.rpcPort = Integer.parseInt(cmd.getOptionValue(RPC_PORT_OPTION));

        if (cmd.hasOption(USER_OPTION)) opts.user = cmd.getOptionValue(USER_OPTION);

        if (cmd.hasOption(PASSWD_OPTION)) opts.passwd = cmd.getOptionValue(PASSWD_OPTION);

        if (cmd.hasOption(INITIAL_HOST_ADDRESS_OPTION)) {
          String[] nodes = cmd.getOptionValue(INITIAL_HOST_ADDRESS_OPTION).split(",");
          try {
            for (String node : nodes) {
              opts.hosts.add(InetAddress.getByName(node.trim()));
            }
          } catch (UnknownHostException e) {
            errorMsg("Unknown host: " + e.getMessage(), options);
          }

        } else {
          System.err.println("Initial hosts must be specified (-d)");
          printUsage(options);
          System.exit(1);
        }

        if (cmd.hasOption(IGNORE_NODES_OPTION)) {
          String[] nodes = cmd.getOptionValue(IGNORE_NODES_OPTION).split(",");
          try {
            for (String node : nodes) {
              opts.ignores.add(InetAddress.getByName(node.trim()));
            }
          } catch (UnknownHostException e) {
            errorMsg("Unknown host: " + e.getMessage(), options);
          }
        }

        // try to load config file first, so that values can be rewritten with other option values.
        // otherwise use default config.
        Config config;
        if (cmd.hasOption(CONFIG_PATH)) {
          File configFile = new File(cmd.getOptionValue(CONFIG_PATH));
          if (!configFile.exists()) {
            errorMsg("Config file not found", options);
          }
          config = new YamlConfigurationLoader().loadConfig(configFile.toURI().toURL());
        } else {
          config = new Config();
        }
        opts.storagePort = config.storage_port;
        opts.sslStoragePort = config.ssl_storage_port;
        opts.throttle = config.stream_throughput_outbound_megabits_per_sec;
        opts.encOptions = config.client_encryption_options;
        opts.serverEncOptions = config.server_encryption_options;

        if (cmd.hasOption(THROTTLE_MBITS)) {
          opts.throttle = Integer.parseInt(cmd.getOptionValue(THROTTLE_MBITS));
        }

        if (cmd.hasOption(SSL_TRUSTSTORE)) {
          opts.encOptions.truststore = cmd.getOptionValue(SSL_TRUSTSTORE);
        }

        if (cmd.hasOption(SSL_TRUSTSTORE_PW)) {
          opts.encOptions.truststore_password = cmd.getOptionValue(SSL_TRUSTSTORE_PW);
        }

        if (cmd.hasOption(SSL_KEYSTORE)) {
          opts.encOptions.keystore = cmd.getOptionValue(SSL_KEYSTORE);
          // if a keystore was provided, lets assume we'll need to use it
          opts.encOptions.require_client_auth = true;
        }

        if (cmd.hasOption(SSL_KEYSTORE_PW)) {
          opts.encOptions.keystore_password = cmd.getOptionValue(SSL_KEYSTORE_PW);
        }

        if (cmd.hasOption(SSL_PROTOCOL)) {
          opts.encOptions.protocol = cmd.getOptionValue(SSL_PROTOCOL);
        }

        if (cmd.hasOption(SSL_ALGORITHM)) {
          opts.encOptions.algorithm = cmd.getOptionValue(SSL_ALGORITHM);
        }

        if (cmd.hasOption(SSL_STORE_TYPE)) {
          opts.encOptions.store_type = cmd.getOptionValue(SSL_STORE_TYPE);
        }

        if (cmd.hasOption(SSL_CIPHER_SUITES)) {
          opts.encOptions.cipher_suites = cmd.getOptionValue(SSL_CIPHER_SUITES).split(",");
        }

        if (cmd.hasOption(TRANSPORT_FACTORY)) {
          ITransportFactory transportFactory =
              getTransportFactory(cmd.getOptionValue(TRANSPORT_FACTORY));
          configureTransportFactory(transportFactory, opts);
          opts.transportFactory = transportFactory;
        }

        return opts;
      } catch (ParseException | ConfigurationException | MalformedURLException e) {
        errorMsg(e.getMessage(), options);
        return null;
      }
    }
Пример #15
0
 public ProgressIndicator() {
   start = lastTime = System.nanoTime();
 }