@After public void tearDown() throws Exception { LOGGER.debug("Doing tearDown"); // Stops the threaded executor JmxCollector is using to poll MBean attribute if (jmxCollector != null) { jmxCollector.destroy(); } if (rrdDb == null) return; if (!rrdDb.isClosed()) { rrdDb.close(); } String path = rrdDb.getPath(); File rrdFile = new File(path); if (rrdFile.exists()) { boolean status = rrdFile.delete(); if (status) { LOGGER.debug("Successfully deleted rrdFile " + path); } else { LOGGER.debug("Unable to delete rrdFile " + path); } } else { LOGGER.debug("rrdFile " + path + " does not exist - cannot delete"); } }
private void createInvalidRrdFile(String expected) throws IOException { // create invalid rrd File rrd = new File(expected); RrdDef rrdDef = new RrdDef(rrd.getAbsolutePath(), 3000); rrdDef.addDatasource("test", DsType.GAUGE, 1, NaN, NaN); rrdDef.addArchive(ConsolFun.AVERAGE, 0.2, 1, 1600); RrdDb r = new RrdDb(rrdDef); r.close(); }
private boolean validateStepSize(File rrdFile) throws IOException { RrdDb r = null; try { r = new RrdDb(rrdFile.getAbsolutePath(), true); return (r.getRrdDef().getStep() == STEP_SIZE); } finally { if (r != null) r.close(); } }
private static Sampleable[] checkDataSources(String rrdPath, Sampleable[] sampleables) throws IOException { RrdDb rrdDb = new RrdDb(rrdPath, true); List<Sampleable> missing = new ArrayList<Sampleable>(); for (Sampleable sampleable : sampleables) { if (rrdDb.getDatasource(sampleable.getName()) == null) { missing.add(sampleable); } } rrdDb.close(); return missing.toArray(new Sampleable[missing.size()]); }
Object execute() throws IOException { String[] words = getRemainingWords(); if (words.length != 2) { throw new IllegalArgumentException("Invalid rrdlast syntax"); } String path = words[1]; RrdDb rrdDb = getRrdDbReference(path); try { long lastUpdateTime = rrdDb.getLastUpdateTime(); println(lastUpdateTime + ""); return new Long(lastUpdateTime); } finally { releaseRrdDbReference(rrdDb); } }
private void collectData(int numRrdStepIterations) throws Exception { String rrdFilename = jmxCollector.getRrdPath(); rrdDb = new RrdDb(rrdFilename); Header header = rrdDb.getHeader(); // Wait for "n" iterations of RRDB's sample rate, then see if MBean value was collected LOGGER.debug("Sleeping for " + (header.getStep() * numRrdStepIterations) + " seconds"); Thread.sleep((header.getStep() * numRrdStepIterations) * 1000); // LOGGER.debug(rrdDb.dump()); long endTime = Calendar.getInstance().getTimeInMillis() / 1000; // +1 because the fetch gets data for times inclusively, e.g., // endTime=12345, so startTime=12345-4=12341, // then fetch data for timestamps 12341, 12342, 12343, 12344, 12345 (which is 5 values) long startTime = endTime - numRrdStepIterations + 1; LOGGER.debug("startTime = " + startTime + ", endTime = " + endTime); FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.TOTAL, startTime, endTime); FetchData fetchData = fetchRequest.fetchData(); double[] values = fetchData.getValues(dataSourceName); assertThat(values.length, is(numRrdStepIterations)); logFetchData(fetchData, "TOTAL"); fetchRequest = rrdDb.createFetchRequest(ConsolFun.AVERAGE, startTime, endTime); fetchData = fetchRequest.fetchData(); values = fetchData.getValues(dataSourceName); assertThat(values.length, is(numRrdStepIterations)); logFetchData(fetchData, "AVERAGE"); fetchRequest = rrdDb.createFetchRequest(ConsolFun.MIN, startTime, endTime); fetchData = fetchRequest.fetchData(); values = fetchData.getValues(dataSourceName); assertThat(values.length, is(numRrdStepIterations)); logFetchData(fetchData, "MIN"); fetchRequest = rrdDb.createFetchRequest(ConsolFun.MAX, startTime, endTime); fetchData = fetchRequest.fetchData(); values = fetchData.getValues(dataSourceName); assertThat(values.length, is(numRrdStepIterations)); logFetchData(fetchData, "MAX"); }
private Map<Long, ArrayList<String>> addRrdData( Map<Long, ArrayList<String>> data, String itemName, ConsolFun consilidationFunction, Date timeBegin, Date timeEnd, long resolution) throws IOException { RrdDb rrdDb = new RrdDb(RRD_FOLDER + File.separator + itemName + ".rrd"); FetchRequest fetchRequest = rrdDb.createFetchRequest( consilidationFunction, Util.getTimestamp(timeBegin), Util.getTimestamp(timeEnd), resolution); FetchData fetchData = fetchRequest.fetchData(); // logger.info(fetchData.toString()); long[] timestamps = fetchData.getTimestamps(); double[][] values = fetchData.getValues(); logger.debug( "RRD fetch returned '{}' rows and '{}' columns", fetchData.getRowCount(), fetchData.getColumnCount()); for (int row = 0; row < fetchData.getRowCount(); row++) { // change to microseconds long time = timestamps[row] * 1000; if (!data.containsKey(time)) { data.put(time, new ArrayList<String>()); } ArrayList<String> vals = data.get(time); int indexOffset = vals.size(); for (int dsIndex = 0; dsIndex < fetchData.getColumnCount(); dsIndex++) { vals.add(dsIndex + indexOffset, formatDouble(values[dsIndex][row], "null", true)); } } rrdDb.close(); return data; }
@Override protected Object doPost(String account, String id, SampleRequest request) throws Exception { File parent = new File(dataDir, account + "/" + id); File file = new File(parent, "database.rrd"); if (!file.exists()) { throw new DatabaseDoesNotExistException(); } RrdDb db = new RrdDb(file.getPath()); Sample sample = db.createSample(Util.normalize(request.getTime(), db.getRrdDef().getStep())); for (Map.Entry<String, Double> entry : request.getValues().entrySet()) { sample.setValue(entry.getKey(), entry.getValue()); } sample.update(); db.close(); return null; }
@Test public void testRrdFileCreationForCounterDataSource() throws Exception { // Set sample rate to 1 sec (default is 60 seconds) so that unit test runs quickly int sampleRate = 1; createJmxCollector("Uptime", JmxCollector.COUNTER_DATA_SOURCE_TYPE, sampleRate); String rrdFilename = jmxCollector.getRrdPath(); assertThat(rrdFilename, is(TEST_DIR + rrdPath)); rrdDb = new RrdDb(rrdFilename); assertThat(rrdDb, not(nullValue())); assertThat(rrdDb.isClosed(), is(false)); Header header = rrdDb.getHeader(); assertThat(header, not(nullValue())); assertThat(header.getStep(), is((long) sampleRate)); assertThat(rrdDb.getDsCount(), is(1)); Datasource dataSource = rrdDb.getDatasource(dataSourceName); assertThat(dataSource, not(nullValue())); DsType dataSourceType = dataSource.getType(); assertThat(dataSourceType, is(DsType.COUNTER)); assertThat(rrdDb.getArcCount(), is(8)); Archive archive = rrdDb.getArchive(ConsolFun.AVERAGE, 1); assertThat(archive, not(nullValue())); assertThat(archive.getRows(), is(60)); archive = rrdDb.getArchive(ConsolFun.AVERAGE, 15); assertThat(archive, not(nullValue())); assertThat(archive.getRows(), is(JmxCollector.ONE_YEAR_IN_15_MINUTE_STEPS)); archive = rrdDb.getArchive(ConsolFun.TOTAL, 1); assertThat(archive, not(nullValue())); assertThat(archive.getRows(), is(60)); archive = rrdDb.getArchive(ConsolFun.TOTAL, 15); assertThat(archive, not(nullValue())); assertThat(archive.getRows(), is(JmxCollector.ONE_YEAR_IN_15_MINUTE_STEPS)); // LOGGER.debug(rrdDb.dump()); }
@Test @Ignore public void testRrdFileCreationWhenRrdFileAlreadyExists() throws Exception { // Set sample rate to 1 sec (default is 60 seconds) so that unit test runs quickly int sampleRate = 1; createJmxCollector("Uptime", JmxCollector.COUNTER_DATA_SOURCE_TYPE, sampleRate); String rrdFilename1 = jmxCollector.getRrdPath(); assertThat(rrdFilename1, is(TEST_DIR + rrdPath)); rrdDb = new RrdDb(rrdFilename1); assertThat(rrdDb, not(nullValue())); assertThat(rrdDb.isClosed(), is(false)); // Attempt to create again LOGGER.debug("Creating JmxCollector again ..."); dataSourceName = "uptime"; rrdPath = dataSourceName + ".rrd"; JmxCollector jmxCollector2 = new JmxCollector(); jmxCollector2.setMbeanName("java.lang:type=Runtime"); jmxCollector2.setMbeanAttributeName("Uptime"); jmxCollector2.setRrdPath(rrdPath); jmxCollector2.setRrdDataSourceName(dataSourceName); jmxCollector2.setRrdDataSourceType(JmxCollector.COUNTER_DATA_SOURCE_TYPE); jmxCollector2.setSampleRate(sampleRate); jmxCollector2.setMetricsDir(TEST_DIR); // Simulates what Spring beans container would do jmxCollector2.configureCollector(); // Verify the 2 JMX Collectors are using the same RRD file String rrdFilename2 = jmxCollector2.getRrdPath(); assertThat(rrdFilename2, is(TEST_DIR + rrdPath)); assertThat(rrdFilename1, equalTo(rrdFilename2)); jmxCollector2.destroy(); }
/** * Retrieves the RRD stored data for the specified metric over the specified time range. * * @param rrdFilename the name of the RRD file containing the metric's data * @param startTime start time, in seconds since Unix epoch, to fetch metric's data * @param endTime end time, in seconds since Unix epoch, to fetch metric's data * @return domain object containing the metric's sampled data, which consists of the timestamps * and their associated values, and the total count of the sampled data * @throws IOException * @throws MetricsGraphException */ private MetricData getMetricData(String rrdFilename, long startTime, long endTime) throws IOException, MetricsGraphException { LOGGER.trace("ENTERING: getMetricData"); // Create RRD DB in read-only mode for the specified RRD file RrdDb rrdDb = new RrdDb(rrdFilename, true); // Extract the data source (should always only be one data source per RRD file - otherwise we // have a problem) if (rrdDb.getDsCount() != 1) { throw new MetricsGraphException( "Only one data source per RRD file is supported - RRD file " + rrdFilename + " has " + rrdDb.getDsCount() + " data sources."); } // The step (sample) interval that determines how often RRD collects the metric's data long rrdStep = rrdDb.getRrdDef().getStep(); // Retrieve the RRD file's data source type (COUNTER or GAUGE) to determine how (later) // to store the metric's data for presentation. DsType dataSourceType = rrdDb.getDatasource(0).getType(); // Fetch the metric's data from the RRD file for the specified time range FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.TOTAL, startTime, endTime); FetchData fetchData = fetchRequest.fetchData(); long[] timestamps = fetchData.getTimestamps(); double[] values = fetchData.getValues(0); // Done retrieving data from the RRD database - close it, otherwise no one else will // be able to access it later. rrdDb.close(); // The lists of the metric's timestamps and their associated values that have non-"NaN" values List<Long> validTimestamps = new ArrayList<Long>(); List<Double> validValues = new ArrayList<Double>(); long totalCount = 0; MetricData metricData = new MetricData(); if (dataSourceType == DsType.COUNTER) { // Counters are for constantly incrementing data, hence they can // have a summation of their totals metricData.setHasTotalCount(true); for (int i = 0; i < timestamps.length; i++) { // Filter out the RRD values that have not yet been sampled (they will // have been set to NaN as a placeholder when the RRD file was created) if (!Double.toString(values[i]).equals("NaN")) { // RRD averages the collected samples over the step interval. // To "undo" this averaging and get the actual count, need to // multiply the sampled data value by the RRD step interval. double nonAveragedValue = (double) (values[i] * rrdStep); validTimestamps.add(timestamps[i]); validValues.add(nonAveragedValue); totalCount += (long) nonAveragedValue; } } } else if (dataSourceType == DsType.GAUGE) { // Gauges are for data that waxes and wanes, hence no total count metricData.setHasTotalCount(false); for (int i = 0; i < timestamps.length; i++) { // Filter out the RRD values that have not yet been sampled (they will // have been set to NaN as a placeholder when the RRD file was created) if (!Double.toString(values[i]).equals("NaN")) { validTimestamps.add(timestamps[i]); validValues.add(values[i]); } } } metricData.setTimestamps(validTimestamps); metricData.setValues(validValues); metricData.setTotalCount(totalCount); LOGGER.trace("EXITING: getMetricData"); return metricData; }
@Override public byte[] createGraph( String metricName, String rrdFilename, long startTime, long endTime, String verticalAxisLabel, String title) throws IOException, MetricsGraphException { // Create RRD DB in read-only mode for the specified RRD file RrdDb rrdDb = new RrdDb(rrdFilename, true); // Extract the data source (should always only be one data source per RRD file - otherwise we // have a problem) if (rrdDb.getDsCount() != 1) { throw new MetricsGraphException( "Only one data source per RRD file is supported - RRD file " + rrdFilename + " has " + rrdDb.getDsCount() + " data sources."); } // Define attributes of the graph to be created for this metric RrdGraphDef graphDef = new RrdGraphDef(); graphDef.setTimeSpan(startTime, endTime); graphDef.setImageFormat("PNG"); graphDef.setShowSignature(false); graphDef.setStep(60); graphDef.setVerticalLabel(verticalAxisLabel); graphDef.setHeight(500); graphDef.setWidth(1000); graphDef.setTitle(title); DsType dataSourceType = rrdDb.getDatasource(0).getType(); // Determine if the Data Source for this RRD file is a COUNTER or GAUGE // (Need to know this because COUNTER data is averaged across samples and the vertical axis of // the // generated graph by default will show data per rrdStep interval) if (dataSourceType == DsType.COUNTER) { // If we ever needed to adjust the metric's data collected by RRD by the archive step // (which is the rrdStep * archiveSampleCount) this is how to do it. // FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.AVERAGE, // startTime, endTime); // Archive archive = rrdDb.findMatchingArchive(fetchRequest); // long archiveStep = archive.getArcStep(); // LOGGER.debug("archiveStep = " + archiveStep); long rrdStep = rrdDb.getRrdDef().getStep(); LOGGER.debug("rrdStep = " + rrdStep); // Still TBD if we want to graph the AVERAGE data on the same graph // graphDef.comment(metricName + " "); // graphDef.datasource("myAverage", rrdFilename, "data", ConsolFun.AVERAGE); // graphDef.datasource("realAverage", "myAverage," + rrdStep + ",*"); // graphDef.line("realAverage", Color.GREEN, "Average", 2); // Multiplied by the rrdStep to "undo" the automatic averaging that RRD does // when it collects TOTAL data - we want the actual totals for the step, not // the average of the totals. graphDef.datasource("myTotal", rrdFilename, "data", ConsolFun.TOTAL); graphDef.datasource("realTotal", "myTotal," + rrdStep + ",*"); graphDef.line("realTotal", Color.BLUE, convertCamelCase(metricName), 2); // Add some spacing between the graph and the summary stats shown beneath the graph graphDef.comment("\\s"); graphDef.comment("\\s"); graphDef.comment("\\c"); // Average, Min, and Max over all of the TOTAL data - displayed at bottom of the graph graphDef.gprint("realTotal", ConsolFun.AVERAGE, "Average = %.3f%s"); graphDef.gprint("realTotal", ConsolFun.MIN, "Min = %.3f%s"); graphDef.gprint("realTotal", ConsolFun.MAX, "Max = %.3f%s"); } else if (dataSourceType == DsType.GAUGE) { graphDef.datasource("myAverage", rrdFilename, "data", ConsolFun.AVERAGE); graphDef.line("myAverage", Color.RED, convertCamelCase(metricName), 2); // Add some spacing between the graph and the summary stats shown beneath the graph graphDef.comment("\\s"); graphDef.comment("\\s"); graphDef.comment("\\c"); // Average, Min, and Max over all of the AVERAGE data - displayed at bottom of the graph graphDef.gprint("myAverage", ConsolFun.AVERAGE, "Average = %.3f%s"); graphDef.gprint("myAverage", ConsolFun.MIN, "Min = %.3f%s"); graphDef.gprint("myAverage", ConsolFun.MAX, "Max = %.3f%s"); } else { rrdDb.close(); throw new MetricsGraphException( "Unsupported data source type " + dataSourceType.name() + " in RRD file " + rrdFilename + ", only COUNTER and GAUGE data source types supported."); } rrdDb.close(); // Use "-" as filename so that RRD creates the graph only in memory (no file is // created, hence no file locking problems due to race conditions between multiple clients) graphDef.setFilename("-"); RrdGraph graph = new RrdGraph(graphDef); return graph.getRrdGraphInfo().getBytes(); }
@Override public void close() throws IOException { db.close(); }
@BeforeClass public static void createRrd() throws IOException { startTime = start - start % step; endTime = startTime + 200 * step; RrdDef def = new RrdDef(fileName, startTime - 3 * step, step); def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 215); def.addDatasource("bar", DsType.GAUGE, 3000, Double.NaN, Double.NaN); RrdDb db = new RrdDb(def, RrdBackendFactory.getFactory(backend)); db.createSample((startTime - step)).setValue(0, 0.0).update(); long sampleTime = startTime; for (double val : vals) { db.createSample(sampleTime).setValue(0, val).update(); sampleTime += step; } if (dorrdtool) { String xmlfile = System.getProperty("java.io.tmpdir") + File.separator + "variabletest.xml"; String rrdfile = System.getProperty("java.io.tmpdir") + File.separator + "variabletest.rrd"; db.dumpXml(xmlfile); System.out.println("rrdtool restore " + xmlfile + " " + rrdfile); String cmd = "rrdtool graph /dev/null " + String.format("--start=%d ", startTime) + String.format("--end=%d ", endTime) + String.format("DEF:baz=%s:bar:AVERAGE ", rrdfile) + "VDEF:min=baz,MINIMUM " + "PRINT:min:\"mininum %1.15le\" " + "VDEF:max=baz,MAXIMUM " + "PRINT:max:\"maximum %1.15le\" " + "VDEF:avg=baz,AVERAGE " + "PRINT:avg:\"average %1.15le\" " + "VDEF:stdev=baz,STDEV " + "PRINT:stdev:\"stdev %1.15le\" " + "VDEF:first=baz,FIRST " + "PRINT:first:\"first %1.15le\" " + "VDEF:last=baz,LAST " + "PRINT:last:\"last %1.15le\" " + "VDEF:total=baz,TOTAL " + "PRINT:total:\"total %1.15le\" " + "VDEF:percent=baz,95,PERCENT " + "PRINT:percent:\"95-th percent %1.15le\" " + "VDEF:percentnan=baz,95,PERCENTNAN " + "PRINT:percentnan:\"95-th percentnan %1.15le\" " + "VDEF:lslope=baz,LSLSLOPE " + "PRINT:lslope:\"lslope %1.15le\" " + "VDEF:lslint=baz,LSLINT " + "PRINT:lslint:\"lslintn %1.15le\" " + "VDEF:lslcorrel=baz,LSLCORREL " + "PRINT:lslcorrel:\"lslcorrel %1.15le\" "; System.out.println(cmd); long interval = (endTime - startTime) / 3; String cmd2 = "rrdtool graph /dev/null " + String.format("--start=%d ", startTime + interval) + String.format("--end=%d ", endTime - interval) + String.format("DEF:baz=%s:bar:AVERAGE ", rrdfile) + "VDEF:min=baz,MINIMUM " + "PRINT:min:\"mininum %1.15le\" " + "VDEF:max=baz,MAXIMUM " + "PRINT:max:\"maximum %1.15le\" " + "VDEF:avg=baz,AVERAGE " + "PRINT:avg:\"average %1.15le\" " + "VDEF:stdev=baz,STDEV " + "PRINT:stdev:\"stdev %1.15le\" " + "VDEF:first=baz,FIRST " + "PRINT:first:\"first %1.15le\" " + "VDEF:last=baz,LAST " + "PRINT:last:\"last %1.15le\" " + "VDEF:total=baz,TOTAL " + "PRINT:total:\"total %1.15le\" " + "VDEF:percent=baz,95,PERCENT " + "PRINT:percent:\"95-th percent %1.15le\" " + "VDEF:percentnan=baz,95,PERCENTNAN " + "PRINT:percentnan:\"95-th percentnan %1.15le\" " + "VDEF:lslope=baz,LSLSLOPE " + "PRINT:lslope:\"lslope %1.15le\" " + "VDEF:lslint=baz,LSLINT " + "PRINT:lslint:\"lslintn %1.15le\" " + "VDEF:lslcorrel=baz,LSLCORREL " + "PRINT:lslcorrel:\"lslcorrel %1.15le\" "; System.out.println(cmd2); String cmd3 = "rrdtool graph /dev/null " + String.format("--start=%d ", startTime - 10 * step) + String.format("--end=%d ", endTime + 2 * step) + String.format("DEF:baz=%s:bar:AVERAGE ", rrdfile) + "VDEF:min=baz,MINIMUM " + "PRINT:min:\"mininum %1.15le\" " + "VDEF:max=baz,MAXIMUM " + "PRINT:max:\"maximum %1.15le\" " + "VDEF:avg=baz,AVERAGE " + "PRINT:avg:\"average %1.15le\" " + "VDEF:stdev=baz,STDEV " + "PRINT:stdev:\"stdev %1.15le\" " + "VDEF:first=baz,FIRST " + "PRINT:first:\"first %1.15le\" " + "VDEF:last=baz,LAST " + "PRINT:last:\"last %1.15le\" " + "VDEF:total=baz,TOTAL " + "PRINT:total:\"total %1.15le\" " + "VDEF:percent=baz,95,PERCENT " + "PRINT:percent:\"95-th percent %1.15le\" " + "VDEF:percentnan=baz,95,PERCENTNAN " + "PRINT:percentnan:\"95-th percentnan %1.15le\" " + "VDEF:lslope=baz,LSLSLOPE " + "PRINT:lslope:\"lslope %1.15le\" " + "VDEF:lslint=baz,LSLINT " + "PRINT:lslint:\"lslintn %1.15le\" " + "VDEF:lslcorrel=baz,LSLCORREL " + "PRINT:lslcorrel:\"lslcorrel %1.15le\" "; System.out.println(cmd3); } }