public void readyToExit() { // block until all outstanding txns return try { client.drain(); } catch (NoConnectionsException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } if (config.checkdata) summary4qa(); try { client.drain(); } catch (NoConnectionsException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } // close down the client connections try { client.close(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.exit(0); }
/** * Core benchmark code. Connect. Initialize. Run the loop. Cleanup. Print Results. * * @throws Exception if anything unexpected happens. */ public void runBenchmark() throws Exception { printHeading("Setup & Initialization"); // connect to one or more servers, loop until success connect(config.servers); // initialize using synchronous call System.out.println("\nPre-loading Tables...\n"); initialize(); // ensure all data from initialize is committed before proceeding to iterate(). client.drain(); // Run the benchmark loop for the requested warmup time // The throughput may be throttled depending on client configuration System.out.println("Warming up for the specified " + config.warmup + " seconds..."); final long warmupEndTime = System.currentTimeMillis() + (1000l * config.warmup); while (warmupEndTime > System.currentTimeMillis()) { iterate(); } printHeading("Starting Benchmark"); // reset the stats after warmup fullStatsContext.fetchAndResetBaseline(); periodicStatsContext.fetchAndResetBaseline(); // print periodic statistics to the console benchmarkStartTS = System.currentTimeMillis(); schedulePeriodicStats(); // Run the benchmark loop for the requested duration // The throughput may be throttled depending on client configuration System.out.println("\nRunning benchmark..."); final long benchmarkEndTime = System.currentTimeMillis() + (1000l * config.duration); while (benchmarkEndTime > System.currentTimeMillis()) { iterate(); } // cancel periodic stats printing timer.cancel(); // block until all outstanding txns return client.drain(); // print the summary results printResults(); // close down the client connections client.close(); }
/** * add 20 shuffled rows * * @throws InterruptedException */ private void load(Client client) throws NoConnectionsException, ProcCallException, IOException, InterruptedException { int pkey = 0; a_int.clear(); a_inline_str.clear(); a_pool_str.clear(); boolean async = true; for (int i = 0; i < 20; i++) { a_int.add(i); a_inline_str.add("a_" + i); a_pool_str.add(simpleString + i); } Collections.shuffle(a_int); Collections.shuffle(a_inline_str); Collections.shuffle(a_pool_str); for (int i = 0; i < 20; i++) { SyncCallback cb = new SyncCallback(); client.callProcedure( cb, "InsertO1", pkey, a_int.get(i), a_inline_str.get(i), a_pool_str.get(i)); if (!async) { cb.waitForResponse(); VoltTable vt = cb.getResponse().getResults()[0]; assertTrue(vt.getRowCount() == 1); } pkey = pkey + 1; } client.drain(); }
public void exitOnError(String err) { prt(err); // cancel periodic stats printing timer.cancel(); // block until all outstanding txns return try { client.drain(); } catch (NoConnectionsException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } // print the summary results try { printResults(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } // close down the client connections try { client.close(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
public void closeClient() { // cancel periodic stats printing try { timer.cancel(); } catch (Exception e) { String msg = "In closeOldClient exception = '" + e.getMessage() + "'."; prt(msg); } try { client.drain(); } catch (NoConnectionsException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } // close down the client connections try { client.close(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
public void testSneakingInAProc() throws IOException { System.out.println("STARTING testSneakingInAProc"); Client client = getClient(); int ctr = 0; for (int i = 0; i < 10; i++) { client.callProcedure(new MPCallback(), "MultiPartition"); ctr++; client.callProcedure(new SPCallback(), "SinglePartition", ctr, ctr); ctr++; client.callProcedure(new SPCallback(), "SinglePartition", ctr, ctr); ctr++; client.callProcedure(new SPCallback(), "SinglePartition", ctr, ctr); ctr++; client.callProcedure(new SPCallback(), "SinglePartition", ctr, ctr); ctr++; } answersReceived.addAndGet(ctr); client.drain(); while (answersReceived.get() > 0) { try { Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); } } }
public void testRejoinInlineStringBug() throws Exception { VoltProjectBuilder builder = getBuilderForTest(); LocalCluster cluster = new LocalCluster("rejoin.jar", 1, 2, 1, BackendTarget.NATIVE_EE_JNI, true); boolean success = cluster.compile(builder); assertTrue(success); MiscUtils.copyFile( builder.getPathToDeployment(), Configuration.getPathToCatalogForTest("rejoin.xml")); cluster.setHasLocalServer(false); cluster.startUp(); Client client; client = ClientFactory.createClient(m_cconfig); client.createConnection("localhost"); ProcedureCallback callback = new ProcedureCallback() { @Override public void clientCallback(ClientResponse clientResponse) throws Exception { if (clientResponse.getStatus() != ClientResponse.SUCCESS) { System.out.println(clientResponse.getStatusString()); } } }; StringBuffer shortBuffer = new StringBuffer(); for (int ii = 0; ii < 33; ii++) { shortBuffer.append('a'); } String shortString = shortBuffer.toString(); StringBuffer longBuffer = new StringBuffer(); for (int ii = 0; ii < 17700; ii++) { longBuffer.append('a'); } String longString = longBuffer.toString(); for (int ii = 0; ii < 119; ii++) { client.callProcedure(callback, "InsertInlinedString", ii, shortString, longString); } shortBuffer.append("aaa"); client.callProcedure(callback, "InsertInlinedString", 120, shortBuffer.toString(), longString); client.drain(); client.close(); cluster.shutDownSingleHost(0); cluster.recoverOne(0, 1, "localhost"); cluster.shutDown(); }
void disconnect() { if (m_client == null) return; try { m_client.drain(); m_client.close(); } catch (Exception e) { e.printStackTrace(); } finally { m_client = null; } }
// duplicate values in A_INT private void loadWithDupes(Client client) throws NoConnectionsException, ProcCallException, IOException { client.callProcedure( new SyncCallback(), "InsertO1", new Long(1), new Long(1), "Jane", "AAAAAA"); client.callProcedure( new SyncCallback(), "InsertO1", new Long(2), new Long(1), "Lily", "BBBBBB"); client.callProcedure( new SyncCallback(), "InsertO1", new Long(3), new Long(1), "Tommy", "CCCCCC"); client.callProcedure( new SyncCallback(), "InsertO1", new Long(4), new Long(2), "Alice", "DDDDDD"); client.callProcedure( new SyncCallback(), "InsertO1", new Long(5), new Long(2), "Mike", "EEEEEE"); client.callProcedure(new SyncCallback(), "InsertO1", new Long(6), new Long(2), "Joe", "FFFFFF"); client.callProcedure( new SyncCallback(), "InsertO1", new Long(7), new Long(2), "Jane", "GGGGGGG"); client.callProcedure( new SyncCallback(), "InsertO1", new Long(8), new Long(2), "Lily", "HHHHHH"); client.drain(); }
/** * Load random data into a partitioned table in VoltDB that has a biging pkey. * * <p>If the VoltTable indicates which column is its pkey, then it will use it, but otherwise it * will assume the first column is the bigint pkey. Note, this works with other integer keys, but * your keyspace is pretty small. * * <p>If mb == 0, then maxRows is used. If maxRows == 0, then mb is used. * * @param table Table with or without schema metadata. * @param mb Target RSS (approximate) * @param maxRows Target maximum rows * @param client To load with. * @param rand To generate random data with. * @param offset Generated pkey values start here. * @param jump Generated pkey values increment by this value. * @throws Exception */ public static void fillTableWithBigintPkey( VoltTable table, int mb, long maxRows, final Client client, Random rand, long offset, long jump) throws Exception { // make sure some kind of limit is set assert ((maxRows > 0) || (mb > 0)); assert (maxRows >= 0); assert (mb >= 0); final int mbTarget = mb > 0 ? mb : Integer.MAX_VALUE; if (maxRows == 0) { maxRows = Long.MAX_VALUE; } System.out.printf( "Filling table %s with rows starting with pkey id %d (every %d rows) until either RSS=%dmb or rowcount=%d\n", table.m_name, offset, jump, mbTarget, maxRows); // find the primary key, assume first col if not found int pkeyColIndex = getBigintPrimaryKeyIndexIfExists(table); if (pkeyColIndex == -1) { pkeyColIndex = 0; assert (table.getColumnType(0).isInteger()); } final AtomicLong rss = new AtomicLong(0); ProcedureCallback insertCallback = new ProcedureCallback() { @Override public void clientCallback(ClientResponse clientResponse) throws Exception { if (clientResponse.getStatus() != ClientResponse.SUCCESS) { System.out.println("Error in loader callback:"); System.out.println(((ClientResponseImpl) clientResponse).toJSONString()); assert (false); } } }; // update the rss value asynchronously final AtomicBoolean rssThreadShouldStop = new AtomicBoolean(false); Thread rssThread = new Thread() { @Override public void run() { long tempRss = rss.get(); long rssPrev = tempRss; while (!rssThreadShouldStop.get()) { tempRss = MiscUtils.getMBRss(client); if (tempRss != rssPrev) { rssPrev = tempRss; rss.set(tempRss); System.out.printf("RSS=%dmb\n", tempRss); // bail when done if (tempRss > mbTarget) { return; } } try { Thread.sleep(2000); } catch (Exception e) { } } } }; // load rows until RSS goal is met (status print every 100k) long i = offset; long rows = 0; rssThread.start(); final String insertProcName = table.m_name.toUpperCase() + ".insert"; while (rss.get() < mbTarget) { Object[] row = randomRow(table, Integer.MAX_VALUE, rand); row[pkeyColIndex] = i; client.callProcedure(insertCallback, insertProcName, row); rows++; if ((rows % 100000) == 0) { System.out.printf("Loading 100000 rows. %d inserts sent (%d max id).\n", rows, i); } // if row limit is set, break if it's hit if (rows >= maxRows) { break; } i += jump; } rssThreadShouldStop.set(true); client.drain(); rssThread.join(); System.out.printf( "Filled table %s with %d rows and now RSS=%dmb\n", table.m_name, rows, rss.get()); }
/** * Core benchmark code. Connect. Initialize. Run the loop. Cleanup. Print Results. * * @throws Exception if anything unexpected happens. */ public void runBenchmark() throws Exception { log.info(HORIZONTAL_RULE); log.info(" Setup & Initialization"); log.info(HORIZONTAL_RULE); final int cidCount = 128; final long[] lastRid = new long[cidCount]; for (int i = 0; i < lastRid.length; i++) { lastRid[i] = 0; } // connect to one or more servers, loop until success connect(); log.info(HORIZONTAL_RULE); log.info("Starting Benchmark"); log.info(HORIZONTAL_RULE); // print periodic statistics to the console benchmarkStartTS = System.currentTimeMillis(); schedulePeriodicStats(); // Run the benchmark loop for the requested duration // The throughput may be throttled depending on client configuration log.info("Running benchmark..."); BigTableLoader partitionedLoader = new BigTableLoader( client, "bigp", (config.partfillerrowmb * 1024 * 1024) / config.fillerrowsize, config.fillerrowsize); partitionedLoader.start(); BigTableLoader replicatedLoader = new BigTableLoader( client, "bigr", (config.replfillerrowmb * 1024 * 1024) / config.fillerrowsize, config.fillerrowsize); replicatedLoader.start(); ReadThread readThread = new ReadThread(client, config.threads, config.threadoffset); readThread.start(); AdHocMayhemThread adHocMayhemThread = new AdHocMayhemThread(client); adHocMayhemThread.start(); List<ClientThread> clientThreads = new ArrayList<ClientThread>(); for (byte cid = (byte) config.threadoffset; cid < config.threadoffset + config.threads; cid++) { ClientThread clientThread = new ClientThread(cid, c, client, processor); clientThread.start(); clientThreads.add(clientThread); } final long benchmarkEndTime = System.currentTimeMillis() + (1000l * config.duration); while (benchmarkEndTime > System.currentTimeMillis()) { Thread.yield(); } replicatedLoader.shutdown(); partitionedLoader.shutdown(); readThread.shutdown(); adHocMayhemThread.shutdown(); for (ClientThread clientThread : clientThreads) { clientThread.shutdown(); } replicatedLoader.join(); partitionedLoader.join(); readThread.join(); adHocMayhemThread.join(); for (ClientThread clientThread : clientThreads) { clientThread.join(); } // cancel periodic stats printing timer.cancel(); shutdown.set(true); es.shutdownNow(); // block until all outstanding txns return client.drain(); client.close(); }
public void summary4qa() { // print the summary results try { printResults(); } catch (Exception e) { e.printStackTrace(); } System.out.printf("\n\nChecking put count in the database against expected data"); Iterator<String> it = hashMap.keySet().iterator(); if (debug) { System.out.printf("HashMap Size: %10d\n", hashMap.size()); System.out.println("poolsize = " + config.poolsize); } try { int printInterval = (int) hashMap.size() / 5; if (hashMap.size() > 10000) if (slow) printInterval = (int) config.poolsize / 200; int c = 0; String lastkey = "error"; while (it.hasNext()) { String key = it.next().toString(); if (debug && c % printInterval == 0) System.out.printf("in summary4qa() count = %d, key = '%s'\n", c, key); client.callProcedure(new SumCallback(key, c, printInterval), "Get", key); lastkey = key; c++; } if (debug) System.out.printf("In summary4qa, count = %d, Last Key = '%s'\n", c, lastkey); } catch (Exception x) { System.out.println("This is not working: " + x + "\n"); x.printStackTrace(); System.exit(1); } try { client.drain(); } catch (NoConnectionsException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("\n-------------------\nTest Results:\n-------------------\n"); System.out.printf("\n\t%10d\tKV Rows Checked\n", hashMap.size()); System.out.printf("\t%10d\tKV Rows Found in database\n", successfulPutCount.get()); System.out.printf( "\t%10d\tKV Rows Missing in database\n", hashMap.size() - successfulPutCount.get()); System.out.printf( "\n\t%10d\tKV Rows are Correct\n", successfulPutCount.get() - missingPutCount.get()); System.out.printf( "\t%10d\tKV Rows are Incorrect (off by %d Puts)\n", missingPutCount.get(), incorrectPutCount.get()); if (successfulPutCount.get() == hashMap.size() && missingPutCount.get() == 0) { System.out.println("\n-------------------\nGood News: Database Put counts match\n"); } else { System.out.println("\n-------------------\nError! Database Put counts don't match!!\n"); System.exit(1); } }
/** * Core benchmark code. Connect. Initialize. Run the loop. Cleanup. Print Results. * * @throws Exception if anything unexpected happens. */ public void runBenchmark() throws Exception { System.out.print(HORIZONTAL_RULE); System.out.println(" Setup & Initialization"); System.out.println(HORIZONTAL_RULE); // preload keys if requested System.out.println(); if (config.preload) { System.out.println("Preloading data store..."); for (int i = config.preloadLowKey; i < config.poolsize; i++) { client.callProcedure( new NullCallback(), "Put", String.format(processor.KeyFormat, i), processor.generateForStore().getStoreValue()); } client.drain(); System.out.println("Preloading complete.\n"); } if (!config.runbenchmark) { System.out.println("Benchmark run disabled by --runbenchmark option, exiting now"); System.exit(0); } System.out.print(HORIZONTAL_RULE); System.out.println("Starting Benchmark"); System.out.println(HORIZONTAL_RULE); // Run the benchmark loop for the requested warmup time // The throughput may be throttled depending on client configuration if (config.warmup > 0) { System.out.println("Warming up..."); final long warmupEndTime = System.currentTimeMillis() + (1000l * config.warmup); while (warmupEndTime > System.currentTimeMillis()) { // Decide whether to perform a GET or PUT operation if (rand.nextDouble() < config.getputratio) { // Get a key/value pair, asynchronously client.callProcedure( new NullCallback(), "Get", processor.generateRandomKeyForRetrieval()); } else { // Put a key/value pair, asynchronously final PayloadProcessor.Pair pair = processor.generateForStore(); client.callProcedure(new NullCallback(), "Put", pair.Key, pair.getStoreValue()); } } } // reset the stats after warmup fullStatsContext.fetchAndResetBaseline(); periodicStatsContext.fetchAndResetBaseline(); // print periodic statistics to the console benchmarkStartTS = System.currentTimeMillis(); schedulePeriodicStats(); if (totalConnections.get() == 1) // If Volt is running on one node only, no need to run this test on multi-partition config.multisingleratio = 0; // Run the benchmark loop for the requested duration // The throughput may be throttled depending on client configuration System.out.println("\nRunning benchmark..."); final long benchmarkEndTime = System.currentTimeMillis() + (1000l * config.duration); long currentTime = System.currentTimeMillis(); long diff = benchmarkEndTime - currentTime; int i = 1; double mpRand; String msg = ""; while (benchmarkEndTime > currentTime) { if (debug && diff != 0 && diff % 5000.00 == 0 && i % 5 == 0) { msg = "i = " + i + ", Time remaining in seconds: " + diff / 1000l + ", totalConnections = " + totalConnections.get(); prt(msg); i++; } if (totalConnections.get() < 1) { if (debug) { msg = "i = " + i + ", diff = '" + diff + ", totalConnections = " + totalConnections.get() + "\n"; } msg += "All connections are lost! VoltDB could be down!!"; prt(msg); } // Decide whether to perform a GET or PUT operation if (rand.nextDouble() < config.getputratio) { // Get a key/value pair, asynchronously mpRand = rand.nextDouble(); if (mpRand < config.multisingleratio) { if (totalConnections.get() > 1 && config.poolsize > 10000) { slow = true; debug = true; } else debug = false; client.callProcedure( new GetCallback(mpRand), "GetMp", processor.generateRandomKeyForRetrieval()); } else { client.callProcedure( new GetCallback(mpRand), "Get", processor.generateRandomKeyForRetrieval()); } } else { // Put a key/value pair, asynchronously final PayloadProcessor.Pair pair = processor.generateForStore(); mpRand = rand.nextDouble(); if (rand.nextDouble() < config.multisingleratio) { if (totalConnections.get() > 1 && config.poolsize > 10000) { slow = true; debug = true; } else debug = false; client.callProcedure( new PutCallback(pair, mpRand), "PutMp", pair.Key, pair.getStoreValue()); } else { client.callProcedure( new PutCallback(pair, mpRand), "Put", pair.Key, pair.getStoreValue()); } } currentTime = System.currentTimeMillis(); diff = benchmarkEndTime - currentTime; } timer.cancel(); }
/** * Core benchmark code. Connect. Initialize. Run the loop. Cleanup. Print Results. * * @param configPath path to configuration file * @throws ConfigurationException * @throws InterruptedException * @throws IOException * @throws NoConnectionsException */ public void runBenchmark(String configPath) throws ConfigurationException, InterruptedException, NoConnectionsException, IOException { System.out.print(HORIZONTAL_RULE); System.out.println(" Setup & Initialization"); System.out.println(HORIZONTAL_RULE); // Parse the XML test configuration. System.out.printf("Reading configuration file '%s'...\n", configPath); List<QueryTestBase> tests = BenchmarkConfiguration.configureTests(configPath, cliConfig.test); // connect to one or more servers, loop until success connect(cliConfig.servers); System.out.print(HORIZONTAL_RULE); System.out.println("Starting Benchmark"); System.out.println(HORIZONTAL_RULE); // Run the benchmark loop for the requested warmup time // The throughput may be throttled depending on client configuration long queryIndex = 1; if (cliConfig.warmup > 0) { System.out.println("Warming up..."); for (String query : new QueryGenerator(tests, cliConfig.warmup)) { // negative query index means warmup to query tracer client.callProcedure(new CallProcedureCallback(queryIndex * -1, query), "@AdHoc", query); queryIndex++; } } // reset the stats after warmup fullStatsContext.fetchAndResetBaseline(); periodicStatsContext.fetchAndResetBaseline(); // print periodic statistics to the console benchmarkStartTS = System.currentTimeMillis(); long queryElapsedMS = 0; schedulePeriodicStats(); // Run the benchmark for the requested duration. System.out.printf("\nRunning '%s' benchmark...\n", cliConfig.test); queryIndex = 1; for (String query : new QueryGenerator(tests, cliConfig.duration)) { client.callProcedure(new CallProcedureCallback(queryIndex, query), "@AdHoc", query); tracer.logQueryInvoked(queryIndex, query); queryIndex++; } printStatistics(); System.out.println(); long totalElapsedMS = System.currentTimeMillis() - benchmarkStartTS; // cancel periodic stats printing timer.cancel(); // block until all outstanding txns return client.drain(); // close the logfile if it exists tracer.close(); // print the summary results printResults(totalElapsedMS, queryElapsedMS); // close down the client connections client.close(); }