Esempio n. 1
0
  /** Run and periodically report export measurements to file. */
  public void run() {
    boolean alldone;
    do {
      try {
        sleep(sleeptime);
      } catch (InterruptedException e) {
        // do nothing
      }

      alldone = true;

      // terminate this thread when all the worker threads are done
      for (Thread t : _threads) {
        if (t.getState() != Thread.State.TERMINATED) {
          alldone = false;
        }
      }

      try {
        Measurements.getMeasurements().exportMeasurementsPart(exporter);
      } catch (IOException e) {
        e.printStackTrace();
      }
    } while (!alldone);
    exportOverall();
  }
Esempio n. 2
0
 /**
  * Cleanup any state for this DB. Called once per DB instance; there is one DB instance per client
  * thread.
  */
 @Override
 public void cleanup() throws DBException {
   // Get the measurements instance as this is the only client that should
   // count clean up time like an update if client-side buffering is
   // enabled.
   Measurements measurements = Measurements.getMeasurements();
   try {
     long st = System.nanoTime();
     if (bufferedMutator != null) {
       bufferedMutator.close();
     }
     if (currentTable != null) {
       currentTable.close();
     }
     long en = System.nanoTime();
     final String type = clientSideBuffering ? "UPDATE" : "CLEANUP";
     measurements.measure(type, (int) ((en - st) / 1000));
     connection.close();
   } catch (IOException e) {
     throw new DBException(e);
   }
 }
Esempio n. 3
0
  /**
   * Exports the measurements to either sysout or a file using the exporter loaded from conf.
   *
   * @throws IOException Either failed to write to output stream or failed to close it.
   */
  private static void exportMeasurements(Properties props, int opcount, long runtime)
      throws IOException {
    MeasurementsExporter exporter = null;
    try {
      // if no destination file is provided the results will be written to stdout
      OutputStream out;
      String exportFile = props.getProperty("exportfile");
      if (exportFile == null) {
        out = System.out;
      } else {
        out = new FileOutputStream(exportFile);
      }

      // if no exporter is provided the default text one will be used
      String exporterStr =
          props.getProperty(
              "exporter", "com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter");
      try {
        exporter =
            (MeasurementsExporter)
                Class.forName(exporterStr).getConstructor(OutputStream.class).newInstance(out);
      } catch (Exception e) {
        System.err.println(
            "Could not find exporter " + exporterStr + ", will use default text reporter.");
        e.printStackTrace();
        exporter = new TextMeasurementsExporter(out);
      }

      exporter.write("OVERALL", "RunTime(ms)", runtime);
      double throughput = 1000.0 * ((double) opcount) / ((double) runtime);
      exporter.write("OVERALL", "Throughput(ops/sec)", throughput);

      Measurements.getMeasurements().exportMeasurements(exporter);
    } finally {
      if (exporter != null) {
        exporter.close();
      }
    }
  }
Esempio n. 4
0
 public void exportOverall() {
   try {
     Measurements.getMeasurements().exportMeasurementsFinal(exporter);
     long opcount = 0;
     long runtime = 0;
     long recon = 0;
     for (Thread t : _threads) {
       ClientThread ct = (ClientThread) t;
       opcount += ct.getOpsDone();
       if (runtime < ct.getRuntime()) {
         runtime = ct.getRuntime();
       }
       recon += ct.getReconnections();
     }
     exporter.write("OVERALL", "Reconnections", recon);
     exporter.write("OVERALL", "RunTime(ms)", runtime);
     exporter.write("OVERALL", "Operations", opcount);
     double throughput = 1000.0 * ((double) opcount) / ((double) runtime);
     exporter.write("OVERALL", "Throughput(ops/sec)", throughput);
     exporter.close();
   } catch (IOException e) {
     e.printStackTrace();
   }
 }
Esempio n. 5
0
  @SuppressWarnings("unchecked")
  public static void main(String[] args) throws FileNotFoundException {
    String dbname;
    Properties props = new Properties();
    Properties fileprops = new Properties();
    boolean dotransactions = true;
    int threadcount = 1;
    int target = 0;
    boolean status = false;
    String label = "";

    // parse arguments
    int argindex = 0;

    if (args.length == 0) {
      usageMessage();
      System.exit(0);
    }

    while (args[argindex].startsWith("-")) {
      if (args[argindex].compareTo("-threads") == 0) {
        argindex++;
        if (argindex >= args.length) {
          usageMessage();
          System.exit(0);
        }
        int tcount = Integer.parseInt(args[argindex]);
        props.setProperty("threadcount", tcount + "");
        argindex++;
      } else if (args[argindex].compareTo("-target") == 0) {
        argindex++;
        if (argindex >= args.length) {
          usageMessage();
          System.exit(0);
        }
        int ttarget = Integer.parseInt(args[argindex]);
        props.setProperty("target", ttarget + "");
        argindex++;
      } else if (args[argindex].compareTo("-load") == 0) {
        dotransactions = false;
        argindex++;
      } else if (args[argindex].compareTo("-t") == 0) {
        dotransactions = true;
        argindex++;
      } else if (args[argindex].compareTo("-s") == 0) {
        status = true;
        argindex++;
      } else if (args[argindex].compareTo("-db") == 0) {
        argindex++;
        if (argindex >= args.length) {
          usageMessage();
          System.exit(0);
        }
        props.setProperty("db", args[argindex]);
        argindex++;
      } else if (args[argindex].compareTo("-l") == 0) {
        argindex++;
        if (argindex >= args.length) {
          usageMessage();
          System.exit(0);
        }
        label = args[argindex];
        argindex++;
      } else if (args[argindex].compareTo("-P") == 0) {
        argindex++;
        if (argindex >= args.length) {
          usageMessage();
          System.exit(0);
        }
        String propfile = args[argindex];
        argindex++;

        Properties myfileprops = new Properties();
        try {
          myfileprops.load(new FileInputStream(propfile));
        } catch (IOException e) {
          e.printStackTrace();
          System.exit(0);
        }

        // Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5
        for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements(); ) {
          String prop = (String) e.nextElement();

          fileprops.setProperty(prop, myfileprops.getProperty(prop));
        }

      } else if (args[argindex].compareTo("-p") == 0) {
        argindex++;
        if (argindex >= args.length) {
          usageMessage();
          System.exit(0);
        }
        int eq = args[argindex].indexOf('=');
        if (eq < 0) {
          usageMessage();
          System.exit(0);
        }

        String name = args[argindex].substring(0, eq);
        String value = args[argindex].substring(eq + 1);
        props.put(name, value);
        // System.out.println("["+name+"]=["+value+"]");
        argindex++;
      } else {
        System.out.println("Unknown option " + args[argindex]);
        usageMessage();
        System.exit(0);
      }

      if (argindex >= args.length) {
        break;
      }
    }

    if (argindex != args.length) {
      usageMessage();
      System.exit(0);
    }

    // set up logging
    // BasicConfigurator.configure();

    // overwrite file properties with properties from the command line

    // Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5
    for (Enumeration e = props.propertyNames(); e.hasMoreElements(); ) {
      String prop = (String) e.nextElement();
      fileprops.setProperty(prop, props.getProperty(prop));
    }

    props = fileprops;

    if (!checkRequiredProperties(props)) {
      System.exit(0);
    }

    long maxExecutionTime = Integer.parseInt(props.getProperty(MAX_EXECUTION_TIME, "0"));

    // get number of threads, target and db
    threadcount = Integer.parseInt(props.getProperty("threadcount", "1"));
    dbname = props.getProperty("db", "com.yahoo.ycsb.BasicDB");
    target = Integer.parseInt(props.getProperty("target", "0"));

    // compute the target throughput
    double targetperthreadperms = -1;
    if (target > 0) {
      double targetperthread = ((double) target) / ((double) threadcount);
      targetperthreadperms = targetperthread / 1000.0;
    }

    System.out.println("YCSB Client 0.1");
    System.out.print("Command line:");
    for (int i = 0; i < args.length; i++) {
      System.out.print(" " + args[i]);
    }
    System.out.println();
    System.err.println("Loading workload...");

    // show a warning message that creating the workload is taking a while
    // but only do so if it is taking longer than 2 seconds
    // (showing the message right away if the setup wasn't taking very long was confusing people)
    Thread warningthread =
        new Thread() {
          public void run() {
            try {
              sleep(2000);
            } catch (InterruptedException e) {
              return;
            }
            System.err.println(" (might take a few minutes for large data sets)");
          }
        };

    warningthread.start();

    // set up measurements
    Measurements.setProperties(props);

    // load the workload
    ClassLoader classLoader = Client.class.getClassLoader();

    Workload workload = null;

    try {
      Class workloadclass =
          classLoader.loadClass(
              props.getProperty(WORKLOAD_PROPERTY, "com.yahoo.ycsb.workloads.CoreWorkload"));
      workload = (Workload) workloadclass.newInstance();
    } catch (Exception e) {
      e.printStackTrace();
      e.printStackTrace(System.out);
      System.exit(0);
    }

    try {
      workload.init(props);
    } catch (WorkloadException e) {
      e.printStackTrace();
      e.printStackTrace(System.out);
      System.exit(0);
    }

    warningthread.interrupt();

    // run the workload

    System.err.println("Starting test.");

    int opcount;
    if (dotransactions) {
      opcount = Integer.parseInt(props.getProperty(OPERATION_COUNT_PROPERTY, "0"));
    } else {
      if (props.containsKey(INSERT_COUNT_PROPERTY)) {
        opcount = Integer.parseInt(props.getProperty(INSERT_COUNT_PROPERTY, "0"));
      } else {
        opcount = Integer.parseInt(props.getProperty(RECORD_COUNT_PROPERTY, "0"));
      }
    }

    int warmupopcount = Integer.parseInt(props.getProperty(WARMUP_OPERATION_COUNT_PROPERTY, "0"));
    int warmupexectime = Integer.parseInt(props.getProperty(WARMUP_EXECUTION_TIME, "0"));

    if (dotransactions) {
      Vector<Thread> warmupThreads = new Vector<Thread>();
      for (int threadid = 0; threadid < threadcount; threadid++) {
        DB db = null;
        try {
          db = DBFactory.rawDB(dbname, props);
        } catch (UnknownDBException e) {
          System.out.println("Unknown DB " + dbname);
          System.exit(0);
        }
        Thread t =
            new WarmupThread(
                db,
                dotransactions,
                workload,
                props,
                warmupopcount / threadcount,
                targetperthreadperms,
                warmupexectime);
        warmupThreads.add(t);
      }

      for (Thread t : warmupThreads) {
        t.start();
      }

      for (Thread t : warmupThreads) {
        try {
          t.join();
        } catch (InterruptedException e) {

        }
      }
    }

    Vector<Thread> threads = new Vector<Thread>();

    for (int threadid = 0; threadid < threadcount; threadid++) {
      DB db = null;
      try {
        db = DBFactory.wrappedDB(dbname, props);
      } catch (UnknownDBException e) {
        System.out.println("Unknown DB " + dbname);
        System.exit(0);
      }

      // spread the ops out across threads evenly, but add the remainder after dividing to thread
      // zero
      int opsPerThread = opcount / threadcount;
      int threadOps =
          threadid == 0 ? opsPerThread + (opcount - opsPerThread * threadcount) : opsPerThread;
      Thread t =
          new ClientThread(db, dotransactions, workload, props, threadOps, targetperthreadperms);
      threads.add(t);
    }

    StatusThread statusthread = null;

    if (status) {
      boolean standardstatus = false;
      if (props.getProperty("measurementtype", "").compareTo("timeseries") == 0) {
        standardstatus = true;
      }
      statusthread = new StatusThread(threads, label, standardstatus);
      statusthread.start();
    }

    MeasurementsExporter exporter = getExporter(props);

    long exportmeasurementsinterval =
        Long.parseLong(props.getProperty(EXPORT_MEASUREMENTS_INTERVAL, "1000"));

    final ExportMeasurementsThread exportmeasurementsthread =
        new ExportMeasurementsThread(threads, exporter, exportmeasurementsinterval);
    exportmeasurementsthread.start();

    // add hook to export measurements on shutdown
    Thread hook =
        new Thread() {
          public void run() {
            exportmeasurementsthread.exportOverall();
          }
        };
    Runtime.getRuntime().addShutdownHook(hook);

    // start client threads
    for (Thread t : threads) {
      t.start();
    }

    Thread terminator = null;

    if (maxExecutionTime > 0) {
      terminator = new TerminatorThread(maxExecutionTime, threads, workload);
      terminator.start();
    }

    long st = System.currentTimeMillis();
    int opsDone = 0;

    for (Thread t : threads) {
      try {
        t.join();
        opsDone += ((ClientThread) t).getOpsDone();
      } catch (InterruptedException e) {
      }
    }

    long en = System.currentTimeMillis();

    if (terminator != null && !terminator.isInterrupted()) {
      terminator.interrupt();
    }

    if (status) {
      statusthread.interrupt();
    }

    try {
      exportmeasurementsthread.join();
    } catch (InterruptedException e) {
      e.printStackTrace();
      e.printStackTrace(System.out);
    }

    try {
      workload.cleanup();
    } catch (WorkloadException e) {
      e.printStackTrace();
      e.printStackTrace(System.out);
      System.exit(0);
    }

    // try
    // {
    //	exportMeasurements(exporter, opsDone, en - st);
    // } catch (IOException e)
    // {
    //	System.err.println("Could not export measurements, error: " + e.getMessage());
    //	e.printStackTrace();
    //	System.exit(-1);
    // }

    System.exit(0);
  }
Esempio n. 6
0
  /** Run and periodically report status. */
  public void run() {
    long st = System.currentTimeMillis();

    long lasten = st;
    long lasttotalops = 0;

    boolean alldone;
    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS");

    do {
      alldone = true;

      int totalops = 0;

      // terminate this thread when all the worker threads are done
      for (Thread t : _threads) {
        if (t.getState() != Thread.State.TERMINATED) {
          alldone = false;
        }

        ClientThread ct = (ClientThread) t;
        totalops += ct.getOpsDone();
      }

      long en = System.currentTimeMillis();

      long interval = en - st;
      // double throughput=1000.0*((double)totalops)/((double)interval);

      double curthroughput =
          1000.0 * (((double) (totalops - lasttotalops)) / ((double) (en - lasten)));

      lasttotalops = totalops;
      lasten = en;

      DecimalFormat d = new DecimalFormat("#.##");

      if (totalops == 0) {
        System.err.println(
            _label
                + " "
                + (interval / 1000)
                + " sec: "
                + totalops
                + " operations; "
                + Measurements.getMeasurements().getSummary());
      } else {
        System.err.println(
            _label
                + " "
                + (interval / 1000)
                + " sec: "
                + totalops
                + " operations; "
                + d.format(curthroughput)
                + " current ops/sec; "
                + Measurements.getMeasurements().getSummary());
      }

      if (_standardstatus) {
        if (totalops == 0) {
          System.out.println(
              _label
                  + " "
                  + (interval / 1000)
                  + " sec: "
                  + totalops
                  + " operations; "
                  + Measurements.getMeasurements().getSummary());
        } else {
          System.out.println(
              _label
                  + " "
                  + (interval / 1000)
                  + " sec: "
                  + totalops
                  + " operations; "
                  + d.format(curthroughput)
                  + " current ops/sec; "
                  + Measurements.getMeasurements().getSummary());
        }
      }

      try {
        sleep(sleeptime);
      } catch (InterruptedException e) {
        // do nothing
      }

    } while (!alldone);
  }