Beispiel #1
0
  public static void main(String[] args) throws Exception {
    VoltDB.setDefaultTimezone();

    config = new SchemaChangeConfig();
    config.parse("SchemaChangeClient", args);

    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setProcedureCallTimeout(30 * 60 * 1000); // 30 min
    client = ClientFactory.createClient(clientConfig);
    String[] servers = config.servers.split(",");
    for (String server : servers) {
      server = server.trim();
      client.createConnection(server);
    }

    // get the topo
    topo = getCluterTopology(client);

    // kick this off with a random schema
    VoltTable t = catalogChange(null, true);

    for (int i = 0; i < 50; i++) {
      // make sure the table is full and mess around with it
      loadTable(t);

      for (int j = 0; j < 50; j++) {
        String tableName = TableHelper.getTableName(t);

        // deterministically sample some rows
        VoltTable preT = sample(t);
        // System.out.printf("First sample:\n%s\n", preT.toFormattedString());

        // move to an entirely new table or migrated schema
        t = catalogChange(t, (j == 0) && (rand.nextInt(5) == 0));

        // if the table has been migrated, check the data
        if (TableHelper.getTableName(t).equals(tableName)) {
          VoltTable guessT = t.clone(4096 * 1024);
          // System.out.printf("Empty clone:\n%s\n", guessT.toFormattedString());

          TableHelper.migrateTable(preT, guessT);
          // System.out.printf("Java migration:\n%s\n", guessT.toFormattedString());

          // deterministically sample the same rows
          VoltTable postT = sample(t);
          // System.out.printf("Second sample:\n%s\n", postT.toFormattedString());

          postT.resetRowPosition();
          preT.resetRowPosition();
          StringBuilder sb = new StringBuilder();
          if (!TableHelper.deepEqualsWithErrorMsg(postT, guessT, sb)) {
            System.err.println(sb.toString());
            assert (false);
          }
        }
      }
    }

    client.close();
  }
  /**
   * Constructor for benchmark instance. Configures VoltDB client and prints configuration.
   *
   * @param config Parsed & validated CLI options.
   */
  public AsyncBenchmark(KVConfig config) {
    this.config = config;

    ClientConfig clientConfig = new ClientConfig("", "", new StatusListener());

    if (config.autotune) {
      clientConfig.enableAutoTune();
      clientConfig.setAutoTuneTargetInternalLatency(config.latencytarget);
    } else {
      clientConfig.setMaxTransactionsPerSecond(config.ratelimit);
    }
    client = ClientFactory.createClient(clientConfig);

    periodicStatsContext = client.createStatsContext();
    fullStatsContext = client.createStatsContext();

    processor =
        new PayloadProcessor(
            config.keysize,
            config.minvaluesize,
            config.maxvaluesize,
            config.entropy,
            config.poolsize,
            config.usecompression);

    System.out.print(HORIZONTAL_RULE);
    System.out.println(" Command Line Configuration");
    System.out.println(HORIZONTAL_RULE);
    System.out.println(config.getConfigDumpString());
  }
Beispiel #3
0
 public Client getClient(long timeout, ClientAuthHashScheme scheme, boolean useAdmin)
     throws IOException {
   final Random r = new Random();
   String listener = null;
   if (useAdmin) {
     listener = m_config.getAdminAddress(r.nextInt(m_config.getListenerCount()));
   } else {
     listener = m_config.getListenerAddress(r.nextInt(m_config.getListenerCount()));
   }
   ClientConfig config = new ClientConfigForTest(m_username, m_password, scheme);
   config.setConnectionResponseTimeout(timeout);
   config.setProcedureCallTimeout(timeout);
   final Client client = ClientFactory.createClient(config);
   // Use the port generated by LocalCluster if applicable
   try {
     client.createConnection(listener);
   }
   // retry once
   catch (ConnectException e) {
     if (useAdmin) {
       listener = m_config.getAdminAddress(r.nextInt(m_config.getListenerCount()));
     } else {
       listener = m_config.getListenerAddress(r.nextInt(m_config.getListenerCount()));
     }
     client.createConnection(listener);
   }
   m_clients.add(client);
   return client;
 }
Beispiel #4
0
 /**
  * Get a VoltClient instance connected to a specific server driven by the VoltServerConfig
  * instance. Find the server by the config's HostId.
  *
  * @return A VoltClient instance connected to the server driven by the VoltServerConfig instance.
  */
 public Client getClientToHostId(int hostId, long timeout) throws IOException {
   final String listener = m_config.getListenerAddress(hostId);
   ClientConfig config = new ClientConfigForTest(m_username, m_password);
   config.setConnectionResponseTimeout(timeout);
   config.setProcedureCallTimeout(timeout);
   final Client client = ClientFactory.createClient(config);
   try {
     client.createConnection(listener);
   }
   // retry once
   catch (ConnectException e) {
     client.createConnection(listener);
   }
   m_clients.add(client);
   return client;
 }
  static Client createClient() {
    StatusListener statusListener = new StatusListener();
    ClientConfig clientConfig = new ClientConfig("", "", statusListener);
    if (config.autoTune) {
      clientConfig.enableAutoTune();
      clientConfig.setAutoTuneTargetInternalLatency(config.latencyTarget);
    } else {
      clientConfig.setMaxTransactionsPerSecond(config.rateLimit);
    }
    Client client = ClientFactory.createClient(clientConfig);
    clientRef.set(client);

    periodicStatsContext = client.createStatsContext();
    fullStatsContext = client.createStatsContext();

    return client;
  }
Beispiel #6
0
  public void processKafkaMessages() throws Exception {
    // Split server list
    final String[] serverlist = m_config.servers.split(",");

    // Create connection
    final ClientConfig c_config = new ClientConfig(m_config.user, m_config.password);
    c_config.setProcedureCallTimeout(0); // Set procedure all to infinite

    m_client = getClient(c_config, serverlist, m_config.port);

    if (m_config.useSuppliedProcedure) {
      m_loader =
          new CSVTupleDataLoader(
              (ClientImpl) m_client, m_config.procedure, new KafkaBulkLoaderCallback());
    } else {
      m_loader =
          new CSVBulkDataLoader(
              (ClientImpl) m_client, m_config.table, m_config.batch, new KafkaBulkLoaderCallback());
    }
    m_loader.setFlushInterval(m_config.flush, m_config.flush);
    m_consumer =
        new KafkaConsumerConnector(
            m_config.zookeeper,
            m_config.useSuppliedProcedure ? m_config.procedure : m_config.table);
    try {
      m_es = getConsumerExecutor(m_consumer, m_loader);
      if (m_config.useSuppliedProcedure) {
        m_log.info(
            "Kafka Consumer from topic: "
                + m_config.topic
                + " Started using procedure: "
                + m_config.procedure);
      } else {
        m_log.info(
            "Kafka Consumer from topic: "
                + m_config.topic
                + " Started for table: "
                + m_config.table);
      }
      m_es.awaitTermination(365, TimeUnit.DAYS);
    } catch (Exception ex) {
      m_log.error("Error in Kafka Consumer", ex);
      System.exit(-1);
    }
    close();
  }
  /**
   * Constructor for benchmark instance. Configures VoltDB client and prints configuration.
   *
   * @param config Parsed & validated CLI options.
   */
  public BaseBenchmark(BenchmarkConfig config) {
    this.config = config;

    ClientConfig clientConfig =
        new ClientConfig(config.user, config.password, new StatusListener());
    if (config.autotune) {
      clientConfig.enableAutoTune();
      clientConfig.setAutoTuneTargetInternalLatency(config.latencytarget);
    } else {
      clientConfig.setMaxTransactionsPerSecond(config.ratelimit);
    }
    client = ClientFactory.createClient(clientConfig);

    periodicStatsContext = client.createStatsContext();
    fullStatsContext = client.createStatsContext();

    printHeading("Command Line Configuration");
    System.out.println(config.getConfigDumpString());
  }
Beispiel #8
0
 public Client getFullyConnectedClient(long timeout) throws IOException {
   final List<String> listeners = m_config.getListenerAddresses();
   final Random r = new Random();
   ClientConfig config = new ClientConfigForTest(m_username, m_password);
   config.setConnectionResponseTimeout(timeout);
   config.setProcedureCallTimeout(timeout);
   final Client client = ClientFactory.createClient(config);
   for (String listener : listeners) {
     // Use the port generated by LocalCluster if applicable
     try {
       client.createConnection(listener);
     }
     // retry once
     catch (ConnectException e) {
       listener = listeners.get(r.nextInt(listeners.size()));
       client.createConnection(listener);
     }
   }
   m_clients.add(client);
   return client;
 }
Beispiel #9
0
  /**
   * Constructor for benchmark instance. Configures VoltDB client and prints configuration.
   *
   * @param cliConfig Parsed & validated CLI options.
   */
  public Benchmark(BenchmarkConfig cliConfig) {

    this.cliConfig = cliConfig;

    ClientConfig clientConfig = new ClientConfig("", "", new StatusListener());
    // Throttle so that ad hoc queries don't get rejected with "planner not available".
    if (cliConfig.querythrottle > 0) {
      System.out.printf(
          "Throttling maximum outstanding transactions to %d\n", cliConfig.querythrottle);
      clientConfig.setMaxOutstandingTxns(cliConfig.querythrottle);
    }
    client = ClientFactory.createClient(clientConfig);

    periodicStatsContext = client.createStatsContext();
    fullStatsContext = client.createStatsContext();

    tracer = new QueryTracer(cliConfig.querytracefile);

    System.out.print(HORIZONTAL_RULE);
    System.out.println(" Command Line Configuration");
    System.out.println(HORIZONTAL_RULE);
    System.out.println(cliConfig.getConfigDumpString());
  }
Beispiel #10
0
  /**
   * Takes a snapshot of all the tables in the database now and check all the rows in each table to
   * see if they satisfy the constraints. The constraints should be added with the table name and
   * table id 0.
   *
   * <p>Since the snapshot files reside on the servers, we have to copy them over to the client in
   * order to check. This might be an overkill, but the alternative is to ask the user to write
   * stored procedure for each table and execute them on all nodes. That's not significantly better,
   * either.
   *
   * <p>This function blocks. Should only be run at the end.
   *
   * @return true if all tables passed the test, false otherwise.
   */
  protected boolean checkTables() {
    String dir = "/tmp";
    String nonce = "data_verification";
    ClientConfig clientConfig = new ClientConfig(m_username, m_password);
    clientConfig.setExpectedOutgoingMessageSize(getExpectedOutgoingMessageSize());
    clientConfig.setHeavyweight(false);

    Client client = ClientFactory.createClient(clientConfig);
    // Host ID to IP mappings
    LinkedHashMap<Integer, String> hostMappings = new LinkedHashMap<Integer, String>();
    /*
     *  The key is the table name. the first one in the pair is the hostname,
     *  the second one is file name
     */
    LinkedHashMap<String, Pair<String, String>> snapshotMappings =
        new LinkedHashMap<String, Pair<String, String>>();
    boolean isSatisfied = true;

    // Load the native library for loading table from snapshot file
    org.voltdb.EELibraryLoader.loadExecutionEngineLibrary(true);

    try {
      boolean keepTrying = true;
      VoltTable[] response = null;

      client.createConnection(m_host);
      // Only initiate the snapshot if it's the first client
      while (m_id == 0) {
        // Take a snapshot of the database. This call is blocking.
        response = client.callProcedure("@SnapshotSave", dir, nonce, 1).getResults();
        if (response.length != 1
            || !response[0].advanceRow()
            || !response[0].getString("RESULT").equals("SUCCESS")) {
          if (keepTrying && response[0].getString("ERR_MSG").contains("ALREADY EXISTS")) {
            client.callProcedure("@SnapshotDelete", new String[] {dir}, new String[] {nonce});
            keepTrying = false;
            continue;
          }

          System.err.println("Failed to take snapshot");
          return false;
        }

        break;
      }

      // Clients other than the one that initiated the snapshot
      // have to check if the snapshot has completed
      if (m_id > 0) {
        int maxTry = 10;

        while (maxTry-- > 0) {
          boolean found = false;
          response = client.callProcedure("@SnapshotStatus").getResults();
          if (response.length != 2) {
            System.err.println("Failed to get snapshot status");
            return false;
          }
          while (response[0].advanceRow()) {
            if (response[0].getString("NONCE").equals(nonce)) {
              found = true;
              break;
            }
          }

          if (found) {
            // This probably means the snapshot is done
            if (response[0].getLong("END_TIME") > 0) break;
          }

          try {
            Thread.sleep(500);
          } catch (InterruptedException e) {
            return false;
          }
        }
      }

      // Get host ID to hostname mappings
      response = client.callProcedure("@SystemInformation").getResults();
      if (response.length != 1) {
        System.err.println("Failed to get host ID to IP address mapping");
        return false;
      }
      while (response[0].advanceRow()) {
        if (!response[0].getString("KEY").equals("HOSTNAME")) {
          continue;
        }
        hostMappings.put(
            (Integer) response[0].get("HOST_ID", VoltType.INTEGER), response[0].getString("VALUE"));
      }

      /* DUMP THE HOST MAPPINGS:
      System.err.println("\n\nhostMappings: ");
      for (Integer i : hostMappings.keySet()) {
          System.err.println("\tkey: " + i + " value: " + hostMappings.get(i));
      }
      */

      // Do a scan to get all the file names and table names
      response = client.callProcedure("@SnapshotScan", dir).getResults();
      if (response.length != 3) {
        System.err.println("Failed to get snapshot filenames");
        return false;
      }

      // Only copy the snapshot files we just created
      while (response[0].advanceRow()) {
        if (!response[0].getString("NONCE").equals(nonce)) continue;

        String[] tables = response[0].getString("TABLES_REQUIRED").split(",");
        for (String t : tables) {
          snapshotMappings.put(t, null);
        }
        break;
      }

      /* DUMP THE SNAPSHOT MAPPINGS:
      System.err.println("\n\nsnapshotMappings: ");
      for (String i : snapshotMappings.keySet()) {
          System.err.println("\tkey: " + i + " value: " + snapshotMappings.get(i));
      }
      */

      while (response[2].advanceRow()) {
        int id = (Integer) response[2].get("HOST_ID", VoltType.INTEGER);
        String tableName = response[2].getString("TABLE");
        if (!snapshotMappings.containsKey(tableName) || !hostMappings.containsKey(id)) {
          System.err.println("FAILED configuring snapshotMappings for: ");
          System.err.println(
              "snapshottingMapping[" + tableName + "] " + snapshotMappings.get(tableName));
          System.err.println("hostMappings[" + id + "] " + hostMappings.get(id));
          continue;
        }

        snapshotMappings.put(
            tableName, Pair.of(hostMappings.get(id), response[2].getString("NAME")));
      }
    } catch (NoConnectionsException e) {
      e.printStackTrace();
      return false;
    } catch (ProcCallException e) {
      e.printStackTrace();
      return false;
    } catch (UnknownHostException e) {
      e.printStackTrace();
      return false;
    } catch (IOException e) {
      e.printStackTrace();
      return false;
    }

    // Iterate through all the tables
    System.err.println("Checking " + m_tableCheckOrder.size() + " table contraints");
    for (String tableName : m_tableCheckOrder) {
      Pair<String, String> value = snapshotMappings.get(tableName);
      if (value == null) {
        System.err.println("No snapshot mapping for table: " + tableName);
        continue;
      }

      String hostName = value.getFirst();
      File file = new File(dir, value.getSecond());
      FileInputStream inputStream = null;
      TableSaveFile saveFile = null;
      long rowCount = 0;

      Pair<String, Integer> key = Pair.of(tableName, 0);
      if (!m_constraints.containsKey(key) || hostName == null) {
        System.err.println("No constraint for : " + tableName);
        continue;
      }

      // Copy the file over
      String localhostName = ConnectionUtil.getHostnameOrAddress();
      final SSHTools ssh = new SSHTools(m_username);
      if (!hostName.equals("localhost") && !hostName.equals(localhostName)) {
        if (!ssh.copyFromRemote(file.getPath(), hostName, file.getPath())) {
          System.err.println(
              "Failed to copy the snapshot file " + file.getPath() + " from host " + hostName);
          return false;
        }
      }

      if (!file.exists()) {
        System.err.println(
            "Snapshot file "
                + file.getPath()
                + " cannot be copied from "
                + hostName
                + " to localhost");
        return false;
      }

      try {
        try {
          inputStream = new FileInputStream(file);
          saveFile = new TableSaveFile(inputStream.getChannel(), 3, null);

          // Get chunks from table
          while (isSatisfied && saveFile.hasMoreChunks()) {
            final BBContainer chunk = saveFile.getNextChunk();
            VoltTable table = null;

            // This probably should not happen
            if (chunk == null) continue;

            table = PrivateVoltTableFactory.createVoltTableFromBuffer(chunk.b, true);
            // Now, check each row
            while (isSatisfied && table.advanceRow()) {
              isSatisfied = Verification.checkRow(m_constraints.get(key), table);
              rowCount++;
            }
            // Release the memory of the chunk we just examined, be good
            chunk.discard();
          }
        } finally {
          if (saveFile != null) {
            saveFile.close();
          }
          if (inputStream != null) inputStream.close();
          if (!hostName.equals("localhost") && !hostName.equals(localhostName) && !file.delete())
            System.err.println("Failed to delete snapshot file " + file.getPath());
        }
      } catch (FileNotFoundException e) {
        e.printStackTrace();
        return false;
      } catch (IOException e) {
        e.printStackTrace();
        return false;
      }

      if (isSatisfied) {
        System.err.println("Table " + tableName + " with " + rowCount + " rows passed check");
      } else {
        System.err.println("Table " + tableName + " failed check");
        break;
      }
    }

    // Clean up the snapshot we made
    try {
      if (m_id == 0) {
        client
            .callProcedure("@SnapshotDelete", new String[] {dir}, new String[] {nonce})
            .getResults();
      }
    } catch (IOException e) {
      e.printStackTrace();
    } catch (ProcCallException e) {
      e.printStackTrace();
    }

    System.err.println(
        "Table checking finished " + (isSatisfied ? "successfully" : "with failures"));

    return isSatisfied;
  }
Beispiel #11
0
  /**
   * Constructor that initializes the framework portions of the client. Creates a Volt client and
   * connects it to all the hosts provided on the command line with the specified username and
   * password
   *
   * @param args
   */
  public ClientMain(String args[]) {
    /*
     * Input parameters: HOST=host:port (may occur multiple times)
     * USER=username PASSWORD=password
     */

    // default values
    String username = "";
    String password = "";
    ControlState state = ControlState.PREPARING; // starting state
    String reason = ""; // and error string
    int transactionRate = -1;
    int id = 0;
    boolean exitOnCompletion = true;
    float checkTransaction = 0;
    boolean checkTables = false;
    String statsDatabaseURL = null;
    String deploymentFilePath = null;
    int statsPollInterval = 10000;
    Integer maxOutstanding = null;

    // scan the inputs once to read everything but host names
    for (final String arg : args) {
      final String[] parts = arg.split("=", 2);
      if (parts.length == 1) {
        state = ControlState.ERROR;
        reason = "Invalid parameter: " + arg;
        break;
      } else if (parts[1].startsWith("${")) {
        continue;
      } else if (parts[0].equals("USER")) {
        username = parts[1];
      } else if (parts[0].equals("PASSWORD")) {
        password = parts[1];
      } else if (parts[0].equals("EXITONCOMPLETION")) {
        exitOnCompletion = Boolean.parseBoolean(parts[1]);
      } else if (parts[0].equals("TXNRATE")) {
        transactionRate = Integer.parseInt(parts[1]);
      } else if (parts[0].equals("ID")) {
        id = Integer.parseInt(parts[1]);
      } else if (parts[0].equals("CHECKTRANSACTION")) {
        checkTransaction = Float.parseFloat(parts[1]);
      } else if (parts[0].equals("CHECKTABLES")) {
        checkTables = Boolean.parseBoolean(parts[1]);
      } else if (parts[0].equals("STATSDATABASEURL")) {
        statsDatabaseURL = parts[1];
      } else if (parts[0].equals("STATSPOLLINTERVAL")) {
        statsPollInterval = Integer.parseInt(parts[1]);
      } else if (parts[0].equals("DEPLOYMENTFILEPATH")) {
        deploymentFilePath = parts[1];
      } else if (parts[0].equals("MAXOUTSTANDING")) {
        maxOutstanding = Integer.parseInt(parts[1]);
      }
    }
    StatsUploaderSettings statsSettings = null;
    if (statsDatabaseURL != null) {
      try {
        statsSettings =
            new StatsUploaderSettings(
                statsDatabaseURL, getApplicationName(), getSubApplicationName(), statsPollInterval);
      } catch (Exception e) {
        System.err.println(e.getMessage());
        // e.printStackTrace();
        statsSettings = null;
      }
    }
    ClientConfig clientConfig = new ClientConfig(username, password);
    clientConfig.setExpectedOutgoingMessageSize(getExpectedOutgoingMessageSize());
    clientConfig.setHeavyweight(useHeavyweightClient());
    clientConfig.setStatsUploaderSettings(statsSettings);
    if (maxOutstanding != null) {
      clientConfig.setMaxOutstandingTxns(maxOutstanding);
    }

    m_voltClient = ClientFactory.createClient(clientConfig);

    m_id = id;
    m_exitOnCompletion = exitOnCompletion;
    m_username = username;
    m_password = password;
    m_txnRate = transactionRate;
    m_txnsPerMillisecond = transactionRate / 1000.0;
    m_deploymentFilePath = deploymentFilePath;

    // report any errors that occurred before the client was instantiated
    if (state != ControlState.PREPARING) setState(state, reason);

    // scan the inputs again looking for host connections
    boolean atLeastOneConnection = false;
    for (final String arg : args) {
      final String[] parts = arg.split("=", 2);
      if (parts.length == 1) {
        continue;
      } else if (parts[0].equals("HOST")) {
        final String hostnport[] = parts[1].split("\\:", 2);
        m_host = hostnport[0];
        String host = hostnport[0];
        int port;
        if (hostnport.length < 2) {
          port = Client.VOLTDB_SERVER_PORT;
        } else {
          port = Integer.valueOf(hostnport[1]);
        }
        try {
          System.err.println("Creating connection to  " + host + ":" + port);
          createConnection(host, port);
          System.err.println("Created connection.");
          atLeastOneConnection = true;
        } catch (final Exception ex) {
          setState(
              ControlState.ERROR, "createConnection to " + arg + " failed: " + ex.getMessage());
        }
      }
    }
    if (!atLeastOneConnection) setState(ControlState.ERROR, "No HOSTS specified on command line.");
    m_checkTransaction = checkTransaction;
    m_checkTables = checkTables;
    m_constraints = new LinkedHashMap<Pair<String, Integer>, Expression>();

    m_countDisplayNames = getTransactionDisplayNames();
    m_counts = new AtomicLong[m_countDisplayNames.length];
    for (int ii = 0; ii < m_counts.length; ii++) {
      m_counts[ii] = new AtomicLong(0);
    }
  }