public void readyToExit() {
// block until all outstanding txns return
try {
client.drain();
} catch (NoConnectionsException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (config.checkdata) summary4qa();
try {
client.drain();
} catch (NoConnectionsException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// close down the client connections
try {
client.close();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.exit(0);
}
/**
* Core benchmark code. Connect. Initialize. Run the loop. Cleanup. Print Results.
*
* @throws Exception if anything unexpected happens.
*/
public void runBenchmark() throws Exception {
printHeading("Setup & Initialization");
// connect to one or more servers, loop until success
connect(config.servers);
// initialize using synchronous call
System.out.println("\nPre-loading Tables...\n");
initialize();
// ensure all data from initialize is committed before proceeding to iterate().
client.drain();
// Run the benchmark loop for the requested warmup time
// The throughput may be throttled depending on client configuration
System.out.println("Warming up for the specified " + config.warmup + " seconds...");
final long warmupEndTime = System.currentTimeMillis() + (1000l * config.warmup);
while (warmupEndTime > System.currentTimeMillis()) {
iterate();
}
printHeading("Starting Benchmark");
// reset the stats after warmup
fullStatsContext.fetchAndResetBaseline();
periodicStatsContext.fetchAndResetBaseline();
// print periodic statistics to the console
benchmarkStartTS = System.currentTimeMillis();
schedulePeriodicStats();
// Run the benchmark loop for the requested duration
// The throughput may be throttled depending on client configuration
System.out.println("\nRunning benchmark...");
final long benchmarkEndTime = System.currentTimeMillis() + (1000l * config.duration);
while (benchmarkEndTime > System.currentTimeMillis()) {
iterate();
}
// cancel periodic stats printing
timer.cancel();
// block until all outstanding txns return
client.drain();
// print the summary results
printResults();
// close down the client connections
client.close();
}
/**
* add 20 shuffled rows
*
* @throws InterruptedException
*/
private void load(Client client)
throws NoConnectionsException, ProcCallException, IOException, InterruptedException {
int pkey = 0;
a_int.clear();
a_inline_str.clear();
a_pool_str.clear();
boolean async = true;
for (int i = 0; i < 20; i++) {
a_int.add(i);
a_inline_str.add("a_" + i);
a_pool_str.add(simpleString + i);
}
Collections.shuffle(a_int);
Collections.shuffle(a_inline_str);
Collections.shuffle(a_pool_str);
for (int i = 0; i < 20; i++) {
SyncCallback cb = new SyncCallback();
client.callProcedure(
cb, "InsertO1", pkey, a_int.get(i), a_inline_str.get(i), a_pool_str.get(i));
if (!async) {
cb.waitForResponse();
VoltTable vt = cb.getResponse().getResults()[0];
assertTrue(vt.getRowCount() == 1);
}
pkey = pkey + 1;
}
client.drain();
}
public void exitOnError(String err) {
prt(err);
// cancel periodic stats printing
timer.cancel();
// block until all outstanding txns return
try {
client.drain();
} catch (NoConnectionsException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// print the summary results
try {
printResults();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// close down the client connections
try {
client.close();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void closeClient() {
// cancel periodic stats printing
try {
timer.cancel();
} catch (Exception e) {
String msg = "In closeOldClient exception = '" + e.getMessage() + "'.";
prt(msg);
}
try {
client.drain();
} catch (NoConnectionsException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// close down the client connections
try {
client.close();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void testSneakingInAProc() throws IOException {
System.out.println("STARTING testSneakingInAProc");
Client client = getClient();
int ctr = 0;
for (int i = 0; i < 10; i++) {
client.callProcedure(new MPCallback(), "MultiPartition");
ctr++;
client.callProcedure(new SPCallback(), "SinglePartition", ctr, ctr);
ctr++;
client.callProcedure(new SPCallback(), "SinglePartition", ctr, ctr);
ctr++;
client.callProcedure(new SPCallback(), "SinglePartition", ctr, ctr);
ctr++;
client.callProcedure(new SPCallback(), "SinglePartition", ctr, ctr);
ctr++;
}
answersReceived.addAndGet(ctr);
client.drain();
while (answersReceived.get() > 0) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
public void testRejoinInlineStringBug() throws Exception {
VoltProjectBuilder builder = getBuilderForTest();
LocalCluster cluster =
new LocalCluster("rejoin.jar", 1, 2, 1, BackendTarget.NATIVE_EE_JNI, true);
boolean success = cluster.compile(builder);
assertTrue(success);
MiscUtils.copyFile(
builder.getPathToDeployment(), Configuration.getPathToCatalogForTest("rejoin.xml"));
cluster.setHasLocalServer(false);
cluster.startUp();
Client client;
client = ClientFactory.createClient(m_cconfig);
client.createConnection("localhost");
ProcedureCallback callback =
new ProcedureCallback() {
@Override
public void clientCallback(ClientResponse clientResponse) throws Exception {
if (clientResponse.getStatus() != ClientResponse.SUCCESS) {
System.out.println(clientResponse.getStatusString());
}
}
};
StringBuffer shortBuffer = new StringBuffer();
for (int ii = 0; ii < 33; ii++) {
shortBuffer.append('a');
}
String shortString = shortBuffer.toString();
StringBuffer longBuffer = new StringBuffer();
for (int ii = 0; ii < 17700; ii++) {
longBuffer.append('a');
}
String longString = longBuffer.toString();
for (int ii = 0; ii < 119; ii++) {
client.callProcedure(callback, "InsertInlinedString", ii, shortString, longString);
}
shortBuffer.append("aaa");
client.callProcedure(callback, "InsertInlinedString", 120, shortBuffer.toString(), longString);
client.drain();
client.close();
cluster.shutDownSingleHost(0);
cluster.recoverOne(0, 1, "localhost");
cluster.shutDown();
}
void disconnect() {
if (m_client == null) return;
try {
m_client.drain();
m_client.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
m_client = null;
}
}
// duplicate values in A_INT
private void loadWithDupes(Client client)
throws NoConnectionsException, ProcCallException, IOException {
client.callProcedure(
new SyncCallback(), "InsertO1", new Long(1), new Long(1), "Jane", "AAAAAA");
client.callProcedure(
new SyncCallback(), "InsertO1", new Long(2), new Long(1), "Lily", "BBBBBB");
client.callProcedure(
new SyncCallback(), "InsertO1", new Long(3), new Long(1), "Tommy", "CCCCCC");
client.callProcedure(
new SyncCallback(), "InsertO1", new Long(4), new Long(2), "Alice", "DDDDDD");
client.callProcedure(
new SyncCallback(), "InsertO1", new Long(5), new Long(2), "Mike", "EEEEEE");
client.callProcedure(new SyncCallback(), "InsertO1", new Long(6), new Long(2), "Joe", "FFFFFF");
client.callProcedure(
new SyncCallback(), "InsertO1", new Long(7), new Long(2), "Jane", "GGGGGGG");
client.callProcedure(
new SyncCallback(), "InsertO1", new Long(8), new Long(2), "Lily", "HHHHHH");
client.drain();
}
/**
* Load random data into a partitioned table in VoltDB that has a biging pkey.
*
* <p>If the VoltTable indicates which column is its pkey, then it will use it, but otherwise it
* will assume the first column is the bigint pkey. Note, this works with other integer keys, but
* your keyspace is pretty small.
*
* <p>If mb == 0, then maxRows is used. If maxRows == 0, then mb is used.
*
* @param table Table with or without schema metadata.
* @param mb Target RSS (approximate)
* @param maxRows Target maximum rows
* @param client To load with.
* @param rand To generate random data with.
* @param offset Generated pkey values start here.
* @param jump Generated pkey values increment by this value.
* @throws Exception
*/
public static void fillTableWithBigintPkey(
VoltTable table,
int mb,
long maxRows,
final Client client,
Random rand,
long offset,
long jump)
throws Exception {
// make sure some kind of limit is set
assert ((maxRows > 0) || (mb > 0));
assert (maxRows >= 0);
assert (mb >= 0);
final int mbTarget = mb > 0 ? mb : Integer.MAX_VALUE;
if (maxRows == 0) {
maxRows = Long.MAX_VALUE;
}
System.out.printf(
"Filling table %s with rows starting with pkey id %d (every %d rows) until either RSS=%dmb or rowcount=%d\n",
table.m_name, offset, jump, mbTarget, maxRows);
// find the primary key, assume first col if not found
int pkeyColIndex = getBigintPrimaryKeyIndexIfExists(table);
if (pkeyColIndex == -1) {
pkeyColIndex = 0;
assert (table.getColumnType(0).isInteger());
}
final AtomicLong rss = new AtomicLong(0);
ProcedureCallback insertCallback =
new ProcedureCallback() {
@Override
public void clientCallback(ClientResponse clientResponse) throws Exception {
if (clientResponse.getStatus() != ClientResponse.SUCCESS) {
System.out.println("Error in loader callback:");
System.out.println(((ClientResponseImpl) clientResponse).toJSONString());
assert (false);
}
}
};
// update the rss value asynchronously
final AtomicBoolean rssThreadShouldStop = new AtomicBoolean(false);
Thread rssThread =
new Thread() {
@Override
public void run() {
long tempRss = rss.get();
long rssPrev = tempRss;
while (!rssThreadShouldStop.get()) {
tempRss = MiscUtils.getMBRss(client);
if (tempRss != rssPrev) {
rssPrev = tempRss;
rss.set(tempRss);
System.out.printf("RSS=%dmb\n", tempRss);
// bail when done
if (tempRss > mbTarget) {
return;
}
}
try {
Thread.sleep(2000);
} catch (Exception e) {
}
}
}
};
// load rows until RSS goal is met (status print every 100k)
long i = offset;
long rows = 0;
rssThread.start();
final String insertProcName = table.m_name.toUpperCase() + ".insert";
while (rss.get() < mbTarget) {
Object[] row = randomRow(table, Integer.MAX_VALUE, rand);
row[pkeyColIndex] = i;
client.callProcedure(insertCallback, insertProcName, row);
rows++;
if ((rows % 100000) == 0) {
System.out.printf("Loading 100000 rows. %d inserts sent (%d max id).\n", rows, i);
}
// if row limit is set, break if it's hit
if (rows >= maxRows) {
break;
}
i += jump;
}
rssThreadShouldStop.set(true);
client.drain();
rssThread.join();
System.out.printf(
"Filled table %s with %d rows and now RSS=%dmb\n", table.m_name, rows, rss.get());
}
/**
* Core benchmark code. Connect. Initialize. Run the loop. Cleanup. Print Results.
*
* @throws Exception if anything unexpected happens.
*/
public void runBenchmark() throws Exception {
log.info(HORIZONTAL_RULE);
log.info(" Setup & Initialization");
log.info(HORIZONTAL_RULE);
final int cidCount = 128;
final long[] lastRid = new long[cidCount];
for (int i = 0; i < lastRid.length; i++) {
lastRid[i] = 0;
}
// connect to one or more servers, loop until success
connect();
log.info(HORIZONTAL_RULE);
log.info("Starting Benchmark");
log.info(HORIZONTAL_RULE);
// print periodic statistics to the console
benchmarkStartTS = System.currentTimeMillis();
schedulePeriodicStats();
// Run the benchmark loop for the requested duration
// The throughput may be throttled depending on client configuration
log.info("Running benchmark...");
BigTableLoader partitionedLoader =
new BigTableLoader(
client,
"bigp",
(config.partfillerrowmb * 1024 * 1024) / config.fillerrowsize,
config.fillerrowsize);
partitionedLoader.start();
BigTableLoader replicatedLoader =
new BigTableLoader(
client,
"bigr",
(config.replfillerrowmb * 1024 * 1024) / config.fillerrowsize,
config.fillerrowsize);
replicatedLoader.start();
ReadThread readThread = new ReadThread(client, config.threads, config.threadoffset);
readThread.start();
AdHocMayhemThread adHocMayhemThread = new AdHocMayhemThread(client);
adHocMayhemThread.start();
List<ClientThread> clientThreads = new ArrayList<ClientThread>();
for (byte cid = (byte) config.threadoffset; cid < config.threadoffset + config.threads; cid++) {
ClientThread clientThread = new ClientThread(cid, c, client, processor);
clientThread.start();
clientThreads.add(clientThread);
}
final long benchmarkEndTime = System.currentTimeMillis() + (1000l * config.duration);
while (benchmarkEndTime > System.currentTimeMillis()) {
Thread.yield();
}
replicatedLoader.shutdown();
partitionedLoader.shutdown();
readThread.shutdown();
adHocMayhemThread.shutdown();
for (ClientThread clientThread : clientThreads) {
clientThread.shutdown();
}
replicatedLoader.join();
partitionedLoader.join();
readThread.join();
adHocMayhemThread.join();
for (ClientThread clientThread : clientThreads) {
clientThread.join();
}
// cancel periodic stats printing
timer.cancel();
shutdown.set(true);
es.shutdownNow();
// block until all outstanding txns return
client.drain();
client.close();
}
public void summary4qa() {
// print the summary results
try {
printResults();
} catch (Exception e) {
e.printStackTrace();
}
System.out.printf("\n\nChecking put count in the database against expected data");
Iterator<String> it = hashMap.keySet().iterator();
if (debug) {
System.out.printf("HashMap Size: %10d\n", hashMap.size());
System.out.println("poolsize = " + config.poolsize);
}
try {
int printInterval = (int) hashMap.size() / 5;
if (hashMap.size() > 10000) if (slow) printInterval = (int) config.poolsize / 200;
int c = 0;
String lastkey = "error";
while (it.hasNext()) {
String key = it.next().toString();
if (debug && c % printInterval == 0)
System.out.printf("in summary4qa() count = %d, key = '%s'\n", c, key);
client.callProcedure(new SumCallback(key, c, printInterval), "Get", key);
lastkey = key;
c++;
}
if (debug) System.out.printf("In summary4qa, count = %d, Last Key = '%s'\n", c, lastkey);
} catch (Exception x) {
System.out.println("This is not working: " + x + "\n");
x.printStackTrace();
System.exit(1);
}
try {
client.drain();
} catch (NoConnectionsException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("\n-------------------\nTest Results:\n-------------------\n");
System.out.printf("\n\t%10d\tKV Rows Checked\n", hashMap.size());
System.out.printf("\t%10d\tKV Rows Found in database\n", successfulPutCount.get());
System.out.printf(
"\t%10d\tKV Rows Missing in database\n", hashMap.size() - successfulPutCount.get());
System.out.printf(
"\n\t%10d\tKV Rows are Correct\n", successfulPutCount.get() - missingPutCount.get());
System.out.printf(
"\t%10d\tKV Rows are Incorrect (off by %d Puts)\n",
missingPutCount.get(), incorrectPutCount.get());
if (successfulPutCount.get() == hashMap.size() && missingPutCount.get() == 0) {
System.out.println("\n-------------------\nGood News: Database Put counts match\n");
} else {
System.out.println("\n-------------------\nError! Database Put counts don't match!!\n");
System.exit(1);
}
}
/**
* Core benchmark code. Connect. Initialize. Run the loop. Cleanup. Print Results.
*
* @throws Exception if anything unexpected happens.
*/
public void runBenchmark() throws Exception {
System.out.print(HORIZONTAL_RULE);
System.out.println(" Setup & Initialization");
System.out.println(HORIZONTAL_RULE);
// preload keys if requested
System.out.println();
if (config.preload) {
System.out.println("Preloading data store...");
for (int i = config.preloadLowKey; i < config.poolsize; i++) {
client.callProcedure(
new NullCallback(),
"Put",
String.format(processor.KeyFormat, i),
processor.generateForStore().getStoreValue());
}
client.drain();
System.out.println("Preloading complete.\n");
}
if (!config.runbenchmark) {
System.out.println("Benchmark run disabled by --runbenchmark option, exiting now");
System.exit(0);
}
System.out.print(HORIZONTAL_RULE);
System.out.println("Starting Benchmark");
System.out.println(HORIZONTAL_RULE);
// Run the benchmark loop for the requested warmup time
// The throughput may be throttled depending on client configuration
if (config.warmup > 0) {
System.out.println("Warming up...");
final long warmupEndTime = System.currentTimeMillis() + (1000l * config.warmup);
while (warmupEndTime > System.currentTimeMillis()) {
// Decide whether to perform a GET or PUT operation
if (rand.nextDouble() < config.getputratio) {
// Get a key/value pair, asynchronously
client.callProcedure(
new NullCallback(), "Get", processor.generateRandomKeyForRetrieval());
} else {
// Put a key/value pair, asynchronously
final PayloadProcessor.Pair pair = processor.generateForStore();
client.callProcedure(new NullCallback(), "Put", pair.Key, pair.getStoreValue());
}
}
}
// reset the stats after warmup
fullStatsContext.fetchAndResetBaseline();
periodicStatsContext.fetchAndResetBaseline();
// print periodic statistics to the console
benchmarkStartTS = System.currentTimeMillis();
schedulePeriodicStats();
if (totalConnections.get() == 1)
// If Volt is running on one node only, no need to run this test on multi-partition
config.multisingleratio = 0;
// Run the benchmark loop for the requested duration
// The throughput may be throttled depending on client configuration
System.out.println("\nRunning benchmark...");
final long benchmarkEndTime = System.currentTimeMillis() + (1000l * config.duration);
long currentTime = System.currentTimeMillis();
long diff = benchmarkEndTime - currentTime;
int i = 1;
double mpRand;
String msg = "";
while (benchmarkEndTime > currentTime) {
if (debug && diff != 0 && diff % 5000.00 == 0 && i % 5 == 0) {
msg =
"i = "
+ i
+ ", Time remaining in seconds: "
+ diff / 1000l
+ ", totalConnections = "
+ totalConnections.get();
prt(msg);
i++;
}
if (totalConnections.get() < 1) {
if (debug) {
msg =
"i = "
+ i
+ ", diff = '"
+ diff
+ ", totalConnections = "
+ totalConnections.get()
+ "\n";
}
msg += "All connections are lost! VoltDB could be down!!";
prt(msg);
}
// Decide whether to perform a GET or PUT operation
if (rand.nextDouble() < config.getputratio) {
// Get a key/value pair, asynchronously
mpRand = rand.nextDouble();
if (mpRand < config.multisingleratio) {
if (totalConnections.get() > 1 && config.poolsize > 10000) {
slow = true;
debug = true;
} else debug = false;
client.callProcedure(
new GetCallback(mpRand), "GetMp", processor.generateRandomKeyForRetrieval());
} else {
client.callProcedure(
new GetCallback(mpRand), "Get", processor.generateRandomKeyForRetrieval());
}
} else {
// Put a key/value pair, asynchronously
final PayloadProcessor.Pair pair = processor.generateForStore();
mpRand = rand.nextDouble();
if (rand.nextDouble() < config.multisingleratio) {
if (totalConnections.get() > 1 && config.poolsize > 10000) {
slow = true;
debug = true;
} else debug = false;
client.callProcedure(
new PutCallback(pair, mpRand), "PutMp", pair.Key, pair.getStoreValue());
} else {
client.callProcedure(
new PutCallback(pair, mpRand), "Put", pair.Key, pair.getStoreValue());
}
}
currentTime = System.currentTimeMillis();
diff = benchmarkEndTime - currentTime;
}
timer.cancel();
}
/**
* Core benchmark code. Connect. Initialize. Run the loop. Cleanup. Print Results.
*
* @param configPath path to configuration file
* @throws ConfigurationException
* @throws InterruptedException
* @throws IOException
* @throws NoConnectionsException
*/
public void runBenchmark(String configPath)
throws ConfigurationException, InterruptedException, NoConnectionsException, IOException {
System.out.print(HORIZONTAL_RULE);
System.out.println(" Setup & Initialization");
System.out.println(HORIZONTAL_RULE);
// Parse the XML test configuration.
System.out.printf("Reading configuration file '%s'...\n", configPath);
List<QueryTestBase> tests = BenchmarkConfiguration.configureTests(configPath, cliConfig.test);
// connect to one or more servers, loop until success
connect(cliConfig.servers);
System.out.print(HORIZONTAL_RULE);
System.out.println("Starting Benchmark");
System.out.println(HORIZONTAL_RULE);
// Run the benchmark loop for the requested warmup time
// The throughput may be throttled depending on client configuration
long queryIndex = 1;
if (cliConfig.warmup > 0) {
System.out.println("Warming up...");
for (String query : new QueryGenerator(tests, cliConfig.warmup)) {
// negative query index means warmup to query tracer
client.callProcedure(new CallProcedureCallback(queryIndex * -1, query), "@AdHoc", query);
queryIndex++;
}
}
// reset the stats after warmup
fullStatsContext.fetchAndResetBaseline();
periodicStatsContext.fetchAndResetBaseline();
// print periodic statistics to the console
benchmarkStartTS = System.currentTimeMillis();
long queryElapsedMS = 0;
schedulePeriodicStats();
// Run the benchmark for the requested duration.
System.out.printf("\nRunning '%s' benchmark...\n", cliConfig.test);
queryIndex = 1;
for (String query : new QueryGenerator(tests, cliConfig.duration)) {
client.callProcedure(new CallProcedureCallback(queryIndex, query), "@AdHoc", query);
tracer.logQueryInvoked(queryIndex, query);
queryIndex++;
}
printStatistics();
System.out.println();
long totalElapsedMS = System.currentTimeMillis() - benchmarkStartTS;
// cancel periodic stats printing
timer.cancel();
// block until all outstanding txns return
client.drain();
// close the logfile if it exists
tracer.close();
// print the summary results
printResults(totalElapsedMS, queryElapsedMS);
// close down the client connections
client.close();
}
