Ejemplo n.º 1
0
  @After
  public void tearDown() throws Exception {
    LOGGER.debug("Doing tearDown");

    // Stops the threaded executor JmxCollector is using to poll MBean attribute
    if (jmxCollector != null) {
      jmxCollector.destroy();
    }

    if (rrdDb == null) return;

    if (!rrdDb.isClosed()) {
      rrdDb.close();
    }
    String path = rrdDb.getPath();
    File rrdFile = new File(path);
    if (rrdFile.exists()) {
      boolean status = rrdFile.delete();
      if (status) {
        LOGGER.debug("Successfully deleted rrdFile " + path);
      } else {
        LOGGER.debug("Unable to delete rrdFile " + path);
      }
    } else {
      LOGGER.debug("rrdFile " + path + " does not exist - cannot delete");
    }
  }
Ejemplo n.º 2
0
 private void createInvalidRrdFile(String expected) throws IOException {
   // create invalid rrd
   File rrd = new File(expected);
   RrdDef rrdDef = new RrdDef(rrd.getAbsolutePath(), 3000);
   rrdDef.addDatasource("test", DsType.GAUGE, 1, NaN, NaN);
   rrdDef.addArchive(ConsolFun.AVERAGE, 0.2, 1, 1600);
   RrdDb r = new RrdDb(rrdDef);
   r.close();
 }
Ejemplo n.º 3
0
 private boolean validateStepSize(File rrdFile) throws IOException {
   RrdDb r = null;
   try {
     r = new RrdDb(rrdFile.getAbsolutePath(), true);
     return (r.getRrdDef().getStep() == STEP_SIZE);
   } finally {
     if (r != null) r.close();
   }
 }
Ejemplo n.º 4
0
 private static Sampleable[] checkDataSources(String rrdPath, Sampleable[] sampleables)
     throws IOException {
   RrdDb rrdDb = new RrdDb(rrdPath, true);
   List<Sampleable> missing = new ArrayList<Sampleable>();
   for (Sampleable sampleable : sampleables) {
     if (rrdDb.getDatasource(sampleable.getName()) == null) {
       missing.add(sampleable);
     }
   }
   rrdDb.close();
   return missing.toArray(new Sampleable[missing.size()]);
 }
Ejemplo n.º 5
0
 Object execute() throws IOException {
   String[] words = getRemainingWords();
   if (words.length != 2) {
     throw new IllegalArgumentException("Invalid rrdlast syntax");
   }
   String path = words[1];
   RrdDb rrdDb = getRrdDbReference(path);
   try {
     long lastUpdateTime = rrdDb.getLastUpdateTime();
     println(lastUpdateTime + "");
     return new Long(lastUpdateTime);
   } finally {
     releaseRrdDbReference(rrdDb);
   }
 }
Ejemplo n.º 6
0
  private void collectData(int numRrdStepIterations) throws Exception {
    String rrdFilename = jmxCollector.getRrdPath();
    rrdDb = new RrdDb(rrdFilename);
    Header header = rrdDb.getHeader();

    // Wait for "n" iterations of RRDB's sample rate, then see if MBean value was collected
    LOGGER.debug("Sleeping for " + (header.getStep() * numRrdStepIterations) + " seconds");
    Thread.sleep((header.getStep() * numRrdStepIterations) * 1000);

    // LOGGER.debug(rrdDb.dump());

    long endTime = Calendar.getInstance().getTimeInMillis() / 1000;

    // +1 because the fetch gets data for times inclusively, e.g.,
    // endTime=12345, so startTime=12345-4=12341,
    // then fetch data for timestamps 12341, 12342, 12343, 12344, 12345 (which is 5 values)
    long startTime = endTime - numRrdStepIterations + 1;
    LOGGER.debug("startTime = " + startTime + ",   endTime = " + endTime);

    FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.TOTAL, startTime, endTime);
    FetchData fetchData = fetchRequest.fetchData();
    double[] values = fetchData.getValues(dataSourceName);
    assertThat(values.length, is(numRrdStepIterations));
    logFetchData(fetchData, "TOTAL");

    fetchRequest = rrdDb.createFetchRequest(ConsolFun.AVERAGE, startTime, endTime);
    fetchData = fetchRequest.fetchData();
    values = fetchData.getValues(dataSourceName);
    assertThat(values.length, is(numRrdStepIterations));
    logFetchData(fetchData, "AVERAGE");

    fetchRequest = rrdDb.createFetchRequest(ConsolFun.MIN, startTime, endTime);
    fetchData = fetchRequest.fetchData();
    values = fetchData.getValues(dataSourceName);
    assertThat(values.length, is(numRrdStepIterations));
    logFetchData(fetchData, "MIN");

    fetchRequest = rrdDb.createFetchRequest(ConsolFun.MAX, startTime, endTime);
    fetchData = fetchRequest.fetchData();
    values = fetchData.getValues(dataSourceName);
    assertThat(values.length, is(numRrdStepIterations));
    logFetchData(fetchData, "MAX");
  }
Ejemplo n.º 7
0
  private Map<Long, ArrayList<String>> addRrdData(
      Map<Long, ArrayList<String>> data,
      String itemName,
      ConsolFun consilidationFunction,
      Date timeBegin,
      Date timeEnd,
      long resolution)
      throws IOException {
    RrdDb rrdDb = new RrdDb(RRD_FOLDER + File.separator + itemName + ".rrd");
    FetchRequest fetchRequest =
        rrdDb.createFetchRequest(
            consilidationFunction,
            Util.getTimestamp(timeBegin),
            Util.getTimestamp(timeEnd),
            resolution);
    FetchData fetchData = fetchRequest.fetchData();
    // logger.info(fetchData.toString());
    long[] timestamps = fetchData.getTimestamps();
    double[][] values = fetchData.getValues();

    logger.debug(
        "RRD fetch returned '{}' rows and '{}' columns",
        fetchData.getRowCount(),
        fetchData.getColumnCount());

    for (int row = 0; row < fetchData.getRowCount(); row++) {
      // change to microseconds
      long time = timestamps[row] * 1000;

      if (!data.containsKey(time)) {
        data.put(time, new ArrayList<String>());
      }
      ArrayList<String> vals = data.get(time);
      int indexOffset = vals.size();
      for (int dsIndex = 0; dsIndex < fetchData.getColumnCount(); dsIndex++) {
        vals.add(dsIndex + indexOffset, formatDouble(values[dsIndex][row], "null", true));
      }
    }
    rrdDb.close();

    return data;
  }
Ejemplo n.º 8
0
  @Override
  protected Object doPost(String account, String id, SampleRequest request) throws Exception {
    File parent = new File(dataDir, account + "/" + id);
    File file = new File(parent, "database.rrd");

    if (!file.exists()) {
      throw new DatabaseDoesNotExistException();
    }

    RrdDb db = new RrdDb(file.getPath());

    Sample sample = db.createSample(Util.normalize(request.getTime(), db.getRrdDef().getStep()));
    for (Map.Entry<String, Double> entry : request.getValues().entrySet()) {
      sample.setValue(entry.getKey(), entry.getValue());
    }
    sample.update();

    db.close();

    return null;
  }
Ejemplo n.º 9
0
  @Test
  public void testRrdFileCreationForCounterDataSource() throws Exception {
    // Set sample rate to 1 sec (default is 60 seconds) so that unit test runs quickly
    int sampleRate = 1;
    createJmxCollector("Uptime", JmxCollector.COUNTER_DATA_SOURCE_TYPE, sampleRate);

    String rrdFilename = jmxCollector.getRrdPath();
    assertThat(rrdFilename, is(TEST_DIR + rrdPath));

    rrdDb = new RrdDb(rrdFilename);
    assertThat(rrdDb, not(nullValue()));
    assertThat(rrdDb.isClosed(), is(false));

    Header header = rrdDb.getHeader();
    assertThat(header, not(nullValue()));
    assertThat(header.getStep(), is((long) sampleRate));

    assertThat(rrdDb.getDsCount(), is(1));
    Datasource dataSource = rrdDb.getDatasource(dataSourceName);
    assertThat(dataSource, not(nullValue()));
    DsType dataSourceType = dataSource.getType();
    assertThat(dataSourceType, is(DsType.COUNTER));

    assertThat(rrdDb.getArcCount(), is(8));

    Archive archive = rrdDb.getArchive(ConsolFun.AVERAGE, 1);
    assertThat(archive, not(nullValue()));
    assertThat(archive.getRows(), is(60));

    archive = rrdDb.getArchive(ConsolFun.AVERAGE, 15);
    assertThat(archive, not(nullValue()));
    assertThat(archive.getRows(), is(JmxCollector.ONE_YEAR_IN_15_MINUTE_STEPS));

    archive = rrdDb.getArchive(ConsolFun.TOTAL, 1);
    assertThat(archive, not(nullValue()));
    assertThat(archive.getRows(), is(60));

    archive = rrdDb.getArchive(ConsolFun.TOTAL, 15);
    assertThat(archive, not(nullValue()));
    assertThat(archive.getRows(), is(JmxCollector.ONE_YEAR_IN_15_MINUTE_STEPS));

    // LOGGER.debug(rrdDb.dump());
  }
Ejemplo n.º 10
0
  @Test
  @Ignore
  public void testRrdFileCreationWhenRrdFileAlreadyExists() throws Exception {
    // Set sample rate to 1 sec (default is 60 seconds) so that unit test runs quickly
    int sampleRate = 1;
    createJmxCollector("Uptime", JmxCollector.COUNTER_DATA_SOURCE_TYPE, sampleRate);

    String rrdFilename1 = jmxCollector.getRrdPath();
    assertThat(rrdFilename1, is(TEST_DIR + rrdPath));

    rrdDb = new RrdDb(rrdFilename1);
    assertThat(rrdDb, not(nullValue()));
    assertThat(rrdDb.isClosed(), is(false));

    // Attempt to create again
    LOGGER.debug("Creating JmxCollector again ...");
    dataSourceName = "uptime";
    rrdPath = dataSourceName + ".rrd";

    JmxCollector jmxCollector2 = new JmxCollector();
    jmxCollector2.setMbeanName("java.lang:type=Runtime");
    jmxCollector2.setMbeanAttributeName("Uptime");
    jmxCollector2.setRrdPath(rrdPath);
    jmxCollector2.setRrdDataSourceName(dataSourceName);
    jmxCollector2.setRrdDataSourceType(JmxCollector.COUNTER_DATA_SOURCE_TYPE);
    jmxCollector2.setSampleRate(sampleRate);
    jmxCollector2.setMetricsDir(TEST_DIR);

    // Simulates what Spring beans container would do
    jmxCollector2.configureCollector();

    // Verify the 2 JMX Collectors are using the same RRD file
    String rrdFilename2 = jmxCollector2.getRrdPath();
    assertThat(rrdFilename2, is(TEST_DIR + rrdPath));
    assertThat(rrdFilename1, equalTo(rrdFilename2));

    jmxCollector2.destroy();
  }
Ejemplo n.º 11
0
  /**
   * Retrieves the RRD stored data for the specified metric over the specified time range.
   *
   * @param rrdFilename the name of the RRD file containing the metric's data
   * @param startTime start time, in seconds since Unix epoch, to fetch metric's data
   * @param endTime end time, in seconds since Unix epoch, to fetch metric's data
   * @return domain object containing the metric's sampled data, which consists of the timestamps
   *     and their associated values, and the total count of the sampled data
   * @throws IOException
   * @throws MetricsGraphException
   */
  private MetricData getMetricData(String rrdFilename, long startTime, long endTime)
      throws IOException, MetricsGraphException {
    LOGGER.trace("ENTERING: getMetricData");

    // Create RRD DB in read-only mode for the specified RRD file
    RrdDb rrdDb = new RrdDb(rrdFilename, true);

    // Extract the data source (should always only be one data source per RRD file - otherwise we
    // have a problem)
    if (rrdDb.getDsCount() != 1) {
      throw new MetricsGraphException(
          "Only one data source per RRD file is supported - RRD file "
              + rrdFilename
              + " has "
              + rrdDb.getDsCount()
              + " data sources.");
    }

    // The step (sample) interval that determines how often RRD collects the metric's data
    long rrdStep = rrdDb.getRrdDef().getStep();

    // Retrieve the RRD file's data source type (COUNTER or GAUGE) to determine how (later)
    // to store the metric's data for presentation.
    DsType dataSourceType = rrdDb.getDatasource(0).getType();

    // Fetch the metric's data from the RRD file for the specified time range
    FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.TOTAL, startTime, endTime);
    FetchData fetchData = fetchRequest.fetchData();
    long[] timestamps = fetchData.getTimestamps();
    double[] values = fetchData.getValues(0);

    // Done retrieving data from the RRD database - close it, otherwise no one else will
    // be able to access it later.
    rrdDb.close();

    // The lists of the metric's timestamps and their associated values that have non-"NaN" values
    List<Long> validTimestamps = new ArrayList<Long>();
    List<Double> validValues = new ArrayList<Double>();

    long totalCount = 0;
    MetricData metricData = new MetricData();

    if (dataSourceType == DsType.COUNTER) {
      // Counters are for constantly incrementing data, hence they can
      // have a summation of their totals
      metricData.setHasTotalCount(true);
      for (int i = 0; i < timestamps.length; i++) {
        // Filter out the RRD values that have not yet been sampled (they will
        // have been set to NaN as a placeholder when the RRD file was created)
        if (!Double.toString(values[i]).equals("NaN")) {
          // RRD averages the collected samples over the step interval.
          // To "undo" this averaging and get the actual count, need to
          // multiply the sampled data value by the RRD step interval.
          double nonAveragedValue = (double) (values[i] * rrdStep);
          validTimestamps.add(timestamps[i]);
          validValues.add(nonAveragedValue);
          totalCount += (long) nonAveragedValue;
        }
      }
    } else if (dataSourceType == DsType.GAUGE) {
      // Gauges are for data that waxes and wanes, hence no total count
      metricData.setHasTotalCount(false);
      for (int i = 0; i < timestamps.length; i++) {
        // Filter out the RRD values that have not yet been sampled (they will
        // have been set to NaN as a placeholder when the RRD file was created)
        if (!Double.toString(values[i]).equals("NaN")) {
          validTimestamps.add(timestamps[i]);
          validValues.add(values[i]);
        }
      }
    }

    metricData.setTimestamps(validTimestamps);
    metricData.setValues(validValues);
    metricData.setTotalCount(totalCount);

    LOGGER.trace("EXITING: getMetricData");

    return metricData;
  }
Ejemplo n.º 12
0
  @Override
  public byte[] createGraph(
      String metricName,
      String rrdFilename,
      long startTime,
      long endTime,
      String verticalAxisLabel,
      String title)
      throws IOException, MetricsGraphException {
    // Create RRD DB in read-only mode for the specified RRD file
    RrdDb rrdDb = new RrdDb(rrdFilename, true);

    // Extract the data source (should always only be one data source per RRD file - otherwise we
    // have a problem)
    if (rrdDb.getDsCount() != 1) {
      throw new MetricsGraphException(
          "Only one data source per RRD file is supported - RRD file "
              + rrdFilename
              + " has "
              + rrdDb.getDsCount()
              + " data sources.");
    }

    // Define attributes of the graph to be created for this metric
    RrdGraphDef graphDef = new RrdGraphDef();
    graphDef.setTimeSpan(startTime, endTime);
    graphDef.setImageFormat("PNG");
    graphDef.setShowSignature(false);
    graphDef.setStep(60);
    graphDef.setVerticalLabel(verticalAxisLabel);
    graphDef.setHeight(500);
    graphDef.setWidth(1000);
    graphDef.setTitle(title);

    DsType dataSourceType = rrdDb.getDatasource(0).getType();

    // Determine if the Data Source for this RRD file is a COUNTER or GAUGE
    // (Need to know this because COUNTER data is averaged across samples and the vertical axis of
    // the
    // generated graph by default will show data per rrdStep interval)
    if (dataSourceType == DsType.COUNTER) {
      // If we ever needed to adjust the metric's data collected by RRD by the archive step
      // (which is the rrdStep * archiveSampleCount) this is how to do it.
      //            FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.AVERAGE,
      // startTime, endTime);
      //            Archive archive = rrdDb.findMatchingArchive(fetchRequest);
      //            long archiveStep = archive.getArcStep();
      //            LOGGER.debug("archiveStep = " + archiveStep);
      long rrdStep = rrdDb.getRrdDef().getStep();
      LOGGER.debug("rrdStep = " + rrdStep);

      // Still TBD if we want to graph the AVERAGE data on the same graph
      //            graphDef.comment(metricName + "   ");
      //            graphDef.datasource("myAverage", rrdFilename, "data", ConsolFun.AVERAGE);
      //            graphDef.datasource("realAverage", "myAverage," + rrdStep + ",*");
      //            graphDef.line("realAverage", Color.GREEN, "Average", 2);

      // Multiplied by the rrdStep to "undo" the automatic averaging that RRD does
      // when it collects TOTAL data - we want the actual totals for the step, not
      // the average of the totals.
      graphDef.datasource("myTotal", rrdFilename, "data", ConsolFun.TOTAL);
      graphDef.datasource("realTotal", "myTotal," + rrdStep + ",*");
      graphDef.line("realTotal", Color.BLUE, convertCamelCase(metricName), 2);

      // Add some spacing between the graph and the summary stats shown beneath the graph
      graphDef.comment("\\s");
      graphDef.comment("\\s");
      graphDef.comment("\\c");

      // Average, Min, and Max over all of the TOTAL data - displayed at bottom of the graph
      graphDef.gprint("realTotal", ConsolFun.AVERAGE, "Average = %.3f%s");
      graphDef.gprint("realTotal", ConsolFun.MIN, "Min = %.3f%s");
      graphDef.gprint("realTotal", ConsolFun.MAX, "Max = %.3f%s");
    } else if (dataSourceType == DsType.GAUGE) {
      graphDef.datasource("myAverage", rrdFilename, "data", ConsolFun.AVERAGE);
      graphDef.line("myAverage", Color.RED, convertCamelCase(metricName), 2);

      // Add some spacing between the graph and the summary stats shown beneath the graph
      graphDef.comment("\\s");
      graphDef.comment("\\s");
      graphDef.comment("\\c");

      // Average, Min, and Max over all of the AVERAGE data - displayed at bottom of the graph
      graphDef.gprint("myAverage", ConsolFun.AVERAGE, "Average = %.3f%s");
      graphDef.gprint("myAverage", ConsolFun.MIN, "Min = %.3f%s");
      graphDef.gprint("myAverage", ConsolFun.MAX, "Max = %.3f%s");
    } else {
      rrdDb.close();
      throw new MetricsGraphException(
          "Unsupported data source type "
              + dataSourceType.name()
              + " in RRD file "
              + rrdFilename
              + ", only COUNTER and GAUGE data source types supported.");
    }

    rrdDb.close();

    // Use "-" as filename so that RRD creates the graph only in memory (no file is
    // created, hence no file locking problems due to race conditions between multiple clients)
    graphDef.setFilename("-");
    RrdGraph graph = new RrdGraph(graphDef);

    return graph.getRrdGraphInfo().getBytes();
  }
Ejemplo n.º 13
0
 @Override
 public void close() throws IOException {
   db.close();
 }
Ejemplo n.º 14
0
  @BeforeClass
  public static void createRrd() throws IOException {
    startTime = start - start % step;
    endTime = startTime + 200 * step;

    RrdDef def = new RrdDef(fileName, startTime - 3 * step, step);

    def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 215);
    def.addDatasource("bar", DsType.GAUGE, 3000, Double.NaN, Double.NaN);

    RrdDb db = new RrdDb(def, RrdBackendFactory.getFactory(backend));

    db.createSample((startTime - step)).setValue(0, 0.0).update();

    long sampleTime = startTime;
    for (double val : vals) {
      db.createSample(sampleTime).setValue(0, val).update();
      sampleTime += step;
    }

    if (dorrdtool) {
      String xmlfile = System.getProperty("java.io.tmpdir") + File.separator + "variabletest.xml";
      String rrdfile = System.getProperty("java.io.tmpdir") + File.separator + "variabletest.rrd";
      db.dumpXml(xmlfile);
      System.out.println("rrdtool restore " + xmlfile + " " + rrdfile);
      String cmd =
          "rrdtool graph /dev/null "
              + String.format("--start=%d ", startTime)
              + String.format("--end=%d ", endTime)
              + String.format("DEF:baz=%s:bar:AVERAGE ", rrdfile)
              + "VDEF:min=baz,MINIMUM "
              + "PRINT:min:\"mininum %1.15le\" "
              + "VDEF:max=baz,MAXIMUM "
              + "PRINT:max:\"maximum %1.15le\" "
              + "VDEF:avg=baz,AVERAGE "
              + "PRINT:avg:\"average %1.15le\" "
              + "VDEF:stdev=baz,STDEV "
              + "PRINT:stdev:\"stdev %1.15le\" "
              + "VDEF:first=baz,FIRST "
              + "PRINT:first:\"first %1.15le\" "
              + "VDEF:last=baz,LAST "
              + "PRINT:last:\"last %1.15le\" "
              + "VDEF:total=baz,TOTAL "
              + "PRINT:total:\"total %1.15le\" "
              + "VDEF:percent=baz,95,PERCENT "
              + "PRINT:percent:\"95-th percent %1.15le\" "
              + "VDEF:percentnan=baz,95,PERCENTNAN "
              + "PRINT:percentnan:\"95-th percentnan %1.15le\" "
              + "VDEF:lslope=baz,LSLSLOPE "
              + "PRINT:lslope:\"lslope %1.15le\" "
              + "VDEF:lslint=baz,LSLINT "
              + "PRINT:lslint:\"lslintn %1.15le\" "
              + "VDEF:lslcorrel=baz,LSLCORREL "
              + "PRINT:lslcorrel:\"lslcorrel %1.15le\" ";
      System.out.println(cmd);
      long interval = (endTime - startTime) / 3;
      String cmd2 =
          "rrdtool graph /dev/null "
              + String.format("--start=%d ", startTime + interval)
              + String.format("--end=%d ", endTime - interval)
              + String.format("DEF:baz=%s:bar:AVERAGE ", rrdfile)
              + "VDEF:min=baz,MINIMUM "
              + "PRINT:min:\"mininum %1.15le\" "
              + "VDEF:max=baz,MAXIMUM "
              + "PRINT:max:\"maximum %1.15le\" "
              + "VDEF:avg=baz,AVERAGE "
              + "PRINT:avg:\"average %1.15le\" "
              + "VDEF:stdev=baz,STDEV "
              + "PRINT:stdev:\"stdev %1.15le\" "
              + "VDEF:first=baz,FIRST "
              + "PRINT:first:\"first %1.15le\" "
              + "VDEF:last=baz,LAST "
              + "PRINT:last:\"last %1.15le\" "
              + "VDEF:total=baz,TOTAL "
              + "PRINT:total:\"total %1.15le\" "
              + "VDEF:percent=baz,95,PERCENT "
              + "PRINT:percent:\"95-th percent %1.15le\" "
              + "VDEF:percentnan=baz,95,PERCENTNAN "
              + "PRINT:percentnan:\"95-th percentnan %1.15le\" "
              + "VDEF:lslope=baz,LSLSLOPE "
              + "PRINT:lslope:\"lslope %1.15le\" "
              + "VDEF:lslint=baz,LSLINT "
              + "PRINT:lslint:\"lslintn %1.15le\" "
              + "VDEF:lslcorrel=baz,LSLCORREL "
              + "PRINT:lslcorrel:\"lslcorrel %1.15le\" ";
      System.out.println(cmd2);

      String cmd3 =
          "rrdtool graph /dev/null "
              + String.format("--start=%d ", startTime - 10 * step)
              + String.format("--end=%d ", endTime + 2 * step)
              + String.format("DEF:baz=%s:bar:AVERAGE ", rrdfile)
              + "VDEF:min=baz,MINIMUM "
              + "PRINT:min:\"mininum %1.15le\" "
              + "VDEF:max=baz,MAXIMUM "
              + "PRINT:max:\"maximum %1.15le\" "
              + "VDEF:avg=baz,AVERAGE "
              + "PRINT:avg:\"average %1.15le\" "
              + "VDEF:stdev=baz,STDEV "
              + "PRINT:stdev:\"stdev %1.15le\" "
              + "VDEF:first=baz,FIRST "
              + "PRINT:first:\"first %1.15le\" "
              + "VDEF:last=baz,LAST "
              + "PRINT:last:\"last %1.15le\" "
              + "VDEF:total=baz,TOTAL "
              + "PRINT:total:\"total %1.15le\" "
              + "VDEF:percent=baz,95,PERCENT "
              + "PRINT:percent:\"95-th percent %1.15le\" "
              + "VDEF:percentnan=baz,95,PERCENTNAN "
              + "PRINT:percentnan:\"95-th percentnan %1.15le\" "
              + "VDEF:lslope=baz,LSLSLOPE "
              + "PRINT:lslope:\"lslope %1.15le\" "
              + "VDEF:lslint=baz,LSLINT "
              + "PRINT:lslint:\"lslintn %1.15le\" "
              + "VDEF:lslcorrel=baz,LSLCORREL "
              + "PRINT:lslcorrel:\"lslcorrel %1.15le\" ";
      System.out.println(cmd3);
    }
  }