/** Run and periodically report export measurements to file. */
public void run() {
boolean alldone;
do {
try {
sleep(sleeptime);
} catch (InterruptedException e) {
// do nothing
}
alldone = true;
// terminate this thread when all the worker threads are done
for (Thread t : _threads) {
if (t.getState() != Thread.State.TERMINATED) {
alldone = false;
}
}
try {
Measurements.getMeasurements().exportMeasurementsPart(exporter);
} catch (IOException e) {
e.printStackTrace();
}
} while (!alldone);
exportOverall();
}
/**
* Cleanup any state for this DB. Called once per DB instance; there is one DB instance per client
* thread.
*/
@Override
public void cleanup() throws DBException {
// Get the measurements instance as this is the only client that should
// count clean up time like an update if client-side buffering is
// enabled.
Measurements measurements = Measurements.getMeasurements();
try {
long st = System.nanoTime();
if (bufferedMutator != null) {
bufferedMutator.close();
}
if (currentTable != null) {
currentTable.close();
}
long en = System.nanoTime();
final String type = clientSideBuffering ? "UPDATE" : "CLEANUP";
measurements.measure(type, (int) ((en - st) / 1000));
connection.close();
} catch (IOException e) {
throw new DBException(e);
}
}
/**
* Exports the measurements to either sysout or a file using the exporter loaded from conf.
*
* @throws IOException Either failed to write to output stream or failed to close it.
*/
private static void exportMeasurements(Properties props, int opcount, long runtime)
throws IOException {
MeasurementsExporter exporter = null;
try {
// if no destination file is provided the results will be written to stdout
OutputStream out;
String exportFile = props.getProperty("exportfile");
if (exportFile == null) {
out = System.out;
} else {
out = new FileOutputStream(exportFile);
}
// if no exporter is provided the default text one will be used
String exporterStr =
props.getProperty(
"exporter", "com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter");
try {
exporter =
(MeasurementsExporter)
Class.forName(exporterStr).getConstructor(OutputStream.class).newInstance(out);
} catch (Exception e) {
System.err.println(
"Could not find exporter " + exporterStr + ", will use default text reporter.");
e.printStackTrace();
exporter = new TextMeasurementsExporter(out);
}
exporter.write("OVERALL", "RunTime(ms)", runtime);
double throughput = 1000.0 * ((double) opcount) / ((double) runtime);
exporter.write("OVERALL", "Throughput(ops/sec)", throughput);
Measurements.getMeasurements().exportMeasurements(exporter);
} finally {
if (exporter != null) {
exporter.close();
}
}
}
public void exportOverall() {
try {
Measurements.getMeasurements().exportMeasurementsFinal(exporter);
long opcount = 0;
long runtime = 0;
long recon = 0;
for (Thread t : _threads) {
ClientThread ct = (ClientThread) t;
opcount += ct.getOpsDone();
if (runtime < ct.getRuntime()) {
runtime = ct.getRuntime();
}
recon += ct.getReconnections();
}
exporter.write("OVERALL", "Reconnections", recon);
exporter.write("OVERALL", "RunTime(ms)", runtime);
exporter.write("OVERALL", "Operations", opcount);
double throughput = 1000.0 * ((double) opcount) / ((double) runtime);
exporter.write("OVERALL", "Throughput(ops/sec)", throughput);
exporter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
@SuppressWarnings("unchecked")
public static void main(String[] args) throws FileNotFoundException {
String dbname;
Properties props = new Properties();
Properties fileprops = new Properties();
boolean dotransactions = true;
int threadcount = 1;
int target = 0;
boolean status = false;
String label = "";
// parse arguments
int argindex = 0;
if (args.length == 0) {
usageMessage();
System.exit(0);
}
while (args[argindex].startsWith("-")) {
if (args[argindex].compareTo("-threads") == 0) {
argindex++;
if (argindex >= args.length) {
usageMessage();
System.exit(0);
}
int tcount = Integer.parseInt(args[argindex]);
props.setProperty("threadcount", tcount + "");
argindex++;
} else if (args[argindex].compareTo("-target") == 0) {
argindex++;
if (argindex >= args.length) {
usageMessage();
System.exit(0);
}
int ttarget = Integer.parseInt(args[argindex]);
props.setProperty("target", ttarget + "");
argindex++;
} else if (args[argindex].compareTo("-load") == 0) {
dotransactions = false;
argindex++;
} else if (args[argindex].compareTo("-t") == 0) {
dotransactions = true;
argindex++;
} else if (args[argindex].compareTo("-s") == 0) {
status = true;
argindex++;
} else if (args[argindex].compareTo("-db") == 0) {
argindex++;
if (argindex >= args.length) {
usageMessage();
System.exit(0);
}
props.setProperty("db", args[argindex]);
argindex++;
} else if (args[argindex].compareTo("-l") == 0) {
argindex++;
if (argindex >= args.length) {
usageMessage();
System.exit(0);
}
label = args[argindex];
argindex++;
} else if (args[argindex].compareTo("-P") == 0) {
argindex++;
if (argindex >= args.length) {
usageMessage();
System.exit(0);
}
String propfile = args[argindex];
argindex++;
Properties myfileprops = new Properties();
try {
myfileprops.load(new FileInputStream(propfile));
} catch (IOException e) {
e.printStackTrace();
System.exit(0);
}
// Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5
for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements(); ) {
String prop = (String) e.nextElement();
fileprops.setProperty(prop, myfileprops.getProperty(prop));
}
} else if (args[argindex].compareTo("-p") == 0) {
argindex++;
if (argindex >= args.length) {
usageMessage();
System.exit(0);
}
int eq = args[argindex].indexOf('=');
if (eq < 0) {
usageMessage();
System.exit(0);
}
String name = args[argindex].substring(0, eq);
String value = args[argindex].substring(eq + 1);
props.put(name, value);
// System.out.println("["+name+"]=["+value+"]");
argindex++;
} else {
System.out.println("Unknown option " + args[argindex]);
usageMessage();
System.exit(0);
}
if (argindex >= args.length) {
break;
}
}
if (argindex != args.length) {
usageMessage();
System.exit(0);
}
// set up logging
// BasicConfigurator.configure();
// overwrite file properties with properties from the command line
// Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5
for (Enumeration e = props.propertyNames(); e.hasMoreElements(); ) {
String prop = (String) e.nextElement();
fileprops.setProperty(prop, props.getProperty(prop));
}
props = fileprops;
if (!checkRequiredProperties(props)) {
System.exit(0);
}
long maxExecutionTime = Integer.parseInt(props.getProperty(MAX_EXECUTION_TIME, "0"));
// get number of threads, target and db
threadcount = Integer.parseInt(props.getProperty("threadcount", "1"));
dbname = props.getProperty("db", "com.yahoo.ycsb.BasicDB");
target = Integer.parseInt(props.getProperty("target", "0"));
// compute the target throughput
double targetperthreadperms = -1;
if (target > 0) {
double targetperthread = ((double) target) / ((double) threadcount);
targetperthreadperms = targetperthread / 1000.0;
}
System.out.println("YCSB Client 0.1");
System.out.print("Command line:");
for (int i = 0; i < args.length; i++) {
System.out.print(" " + args[i]);
}
System.out.println();
System.err.println("Loading workload...");
// show a warning message that creating the workload is taking a while
// but only do so if it is taking longer than 2 seconds
// (showing the message right away if the setup wasn't taking very long was confusing people)
Thread warningthread =
new Thread() {
public void run() {
try {
sleep(2000);
} catch (InterruptedException e) {
return;
}
System.err.println(" (might take a few minutes for large data sets)");
}
};
warningthread.start();
// set up measurements
Measurements.setProperties(props);
// load the workload
ClassLoader classLoader = Client.class.getClassLoader();
Workload workload = null;
try {
Class workloadclass =
classLoader.loadClass(
props.getProperty(WORKLOAD_PROPERTY, "com.yahoo.ycsb.workloads.CoreWorkload"));
workload = (Workload) workloadclass.newInstance();
} catch (Exception e) {
e.printStackTrace();
e.printStackTrace(System.out);
System.exit(0);
}
try {
workload.init(props);
} catch (WorkloadException e) {
e.printStackTrace();
e.printStackTrace(System.out);
System.exit(0);
}
warningthread.interrupt();
// run the workload
System.err.println("Starting test.");
int opcount;
if (dotransactions) {
opcount = Integer.parseInt(props.getProperty(OPERATION_COUNT_PROPERTY, "0"));
} else {
if (props.containsKey(INSERT_COUNT_PROPERTY)) {
opcount = Integer.parseInt(props.getProperty(INSERT_COUNT_PROPERTY, "0"));
} else {
opcount = Integer.parseInt(props.getProperty(RECORD_COUNT_PROPERTY, "0"));
}
}
int warmupopcount = Integer.parseInt(props.getProperty(WARMUP_OPERATION_COUNT_PROPERTY, "0"));
int warmupexectime = Integer.parseInt(props.getProperty(WARMUP_EXECUTION_TIME, "0"));
if (dotransactions) {
Vector<Thread> warmupThreads = new Vector<Thread>();
for (int threadid = 0; threadid < threadcount; threadid++) {
DB db = null;
try {
db = DBFactory.rawDB(dbname, props);
} catch (UnknownDBException e) {
System.out.println("Unknown DB " + dbname);
System.exit(0);
}
Thread t =
new WarmupThread(
db,
dotransactions,
workload,
props,
warmupopcount / threadcount,
targetperthreadperms,
warmupexectime);
warmupThreads.add(t);
}
for (Thread t : warmupThreads) {
t.start();
}
for (Thread t : warmupThreads) {
try {
t.join();
} catch (InterruptedException e) {
}
}
}
Vector<Thread> threads = new Vector<Thread>();
for (int threadid = 0; threadid < threadcount; threadid++) {
DB db = null;
try {
db = DBFactory.wrappedDB(dbname, props);
} catch (UnknownDBException e) {
System.out.println("Unknown DB " + dbname);
System.exit(0);
}
// spread the ops out across threads evenly, but add the remainder after dividing to thread
// zero
int opsPerThread = opcount / threadcount;
int threadOps =
threadid == 0 ? opsPerThread + (opcount - opsPerThread * threadcount) : opsPerThread;
Thread t =
new ClientThread(db, dotransactions, workload, props, threadOps, targetperthreadperms);
threads.add(t);
}
StatusThread statusthread = null;
if (status) {
boolean standardstatus = false;
if (props.getProperty("measurementtype", "").compareTo("timeseries") == 0) {
standardstatus = true;
}
statusthread = new StatusThread(threads, label, standardstatus);
statusthread.start();
}
MeasurementsExporter exporter = getExporter(props);
long exportmeasurementsinterval =
Long.parseLong(props.getProperty(EXPORT_MEASUREMENTS_INTERVAL, "1000"));
final ExportMeasurementsThread exportmeasurementsthread =
new ExportMeasurementsThread(threads, exporter, exportmeasurementsinterval);
exportmeasurementsthread.start();
// add hook to export measurements on shutdown
Thread hook =
new Thread() {
public void run() {
exportmeasurementsthread.exportOverall();
}
};
Runtime.getRuntime().addShutdownHook(hook);
// start client threads
for (Thread t : threads) {
t.start();
}
Thread terminator = null;
if (maxExecutionTime > 0) {
terminator = new TerminatorThread(maxExecutionTime, threads, workload);
terminator.start();
}
long st = System.currentTimeMillis();
int opsDone = 0;
for (Thread t : threads) {
try {
t.join();
opsDone += ((ClientThread) t).getOpsDone();
} catch (InterruptedException e) {
}
}
long en = System.currentTimeMillis();
if (terminator != null && !terminator.isInterrupted()) {
terminator.interrupt();
}
if (status) {
statusthread.interrupt();
}
try {
exportmeasurementsthread.join();
} catch (InterruptedException e) {
e.printStackTrace();
e.printStackTrace(System.out);
}
try {
workload.cleanup();
} catch (WorkloadException e) {
e.printStackTrace();
e.printStackTrace(System.out);
System.exit(0);
}
// try
// {
// exportMeasurements(exporter, opsDone, en - st);
// } catch (IOException e)
// {
// System.err.println("Could not export measurements, error: " + e.getMessage());
// e.printStackTrace();
// System.exit(-1);
// }
System.exit(0);
}
/** Run and periodically report status. */
public void run() {
long st = System.currentTimeMillis();
long lasten = st;
long lasttotalops = 0;
boolean alldone;
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS");
do {
alldone = true;
int totalops = 0;
// terminate this thread when all the worker threads are done
for (Thread t : _threads) {
if (t.getState() != Thread.State.TERMINATED) {
alldone = false;
}
ClientThread ct = (ClientThread) t;
totalops += ct.getOpsDone();
}
long en = System.currentTimeMillis();
long interval = en - st;
// double throughput=1000.0*((double)totalops)/((double)interval);
double curthroughput =
1000.0 * (((double) (totalops - lasttotalops)) / ((double) (en - lasten)));
lasttotalops = totalops;
lasten = en;
DecimalFormat d = new DecimalFormat("#.##");
if (totalops == 0) {
System.err.println(
_label
+ " "
+ (interval / 1000)
+ " sec: "
+ totalops
+ " operations; "
+ Measurements.getMeasurements().getSummary());
} else {
System.err.println(
_label
+ " "
+ (interval / 1000)
+ " sec: "
+ totalops
+ " operations; "
+ d.format(curthroughput)
+ " current ops/sec; "
+ Measurements.getMeasurements().getSummary());
}
if (_standardstatus) {
if (totalops == 0) {
System.out.println(
_label
+ " "
+ (interval / 1000)
+ " sec: "
+ totalops
+ " operations; "
+ Measurements.getMeasurements().getSummary());
} else {
System.out.println(
_label
+ " "
+ (interval / 1000)
+ " sec: "
+ totalops
+ " operations; "
+ d.format(curthroughput)
+ " current ops/sec; "
+ Measurements.getMeasurements().getSummary());
}
}
try {
sleep(sleeptime);
} catch (InterruptedException e) {
// do nothing
}
} while (!alldone);
}