private static Sampleable[] checkDataSources(String rrdPath, Sampleable[] sampleables)
throws IOException {
RrdDb rrdDb = new RrdDb(rrdPath, true);
List<Sampleable> missing = new ArrayList<Sampleable>();
for (Sampleable sampleable : sampleables) {
if (rrdDb.getDatasource(sampleable.getName()) == null) {
missing.add(sampleable);
}
}
rrdDb.close();
return missing.toArray(new Sampleable[missing.size()]);
}
@Test
public void testRrdFileCreationForGaugeDataSource() throws Exception {
// Set sample rate to 1 sec (default is 60 seconds) so that unit test runs quickly
int sampleRate = 1;
createJmxCollector("Uptime", JmxCollector.GAUGE_DATA_SOURCE_TYPE, sampleRate);
String rrdFilename = jmxCollector.getRrdPath();
assertThat(rrdFilename, is(TEST_DIR + rrdPath));
rrdDb = new RrdDb(rrdFilename);
assertThat(rrdDb, not(nullValue()));
assertThat(rrdDb.isClosed(), is(false));
Header header = rrdDb.getHeader();
assertThat(header, not(nullValue()));
assertThat(header.getStep(), is((long) sampleRate)); // verify 60 second sample rate
assertThat(rrdDb.getDsCount(), is(1));
Datasource dataSource = rrdDb.getDatasource(dataSourceName);
assertThat(dataSource, not(nullValue()));
DsType dataSourceType = dataSource.getType();
assertThat(dataSourceType, is(DsType.GAUGE));
assertThat(rrdDb.getArcCount(), is(8));
Archive archive = rrdDb.getArchive(ConsolFun.MIN, 1);
assertThat(archive, not(nullValue()));
assertThat(archive.getRows(), is(60));
archive = rrdDb.getArchive(ConsolFun.MIN, 15);
assertThat(archive, not(nullValue()));
assertThat(archive.getRows(), is(JmxCollector.ONE_YEAR_IN_15_MINUTE_STEPS));
archive = rrdDb.getArchive(ConsolFun.MAX, 1);
assertThat(archive, not(nullValue()));
assertThat(archive.getRows(), is(60));
archive = rrdDb.getArchive(ConsolFun.MAX, 15);
assertThat(archive, not(nullValue()));
assertThat(archive.getRows(), is(JmxCollector.ONE_YEAR_IN_15_MINUTE_STEPS));
archive = rrdDb.getArchive(ConsolFun.AVERAGE, 1);
assertThat(archive, not(nullValue()));
assertThat(archive.getRows(), is(60));
archive = rrdDb.getArchive(ConsolFun.AVERAGE, 15);
assertThat(archive, not(nullValue()));
assertThat(archive.getRows(), is(JmxCollector.ONE_YEAR_IN_15_MINUTE_STEPS));
}
/**
* Retrieves the RRD stored data for the specified metric over the specified time range.
*
* @param rrdFilename the name of the RRD file containing the metric's data
* @param startTime start time, in seconds since Unix epoch, to fetch metric's data
* @param endTime end time, in seconds since Unix epoch, to fetch metric's data
* @return domain object containing the metric's sampled data, which consists of the timestamps
* and their associated values, and the total count of the sampled data
* @throws IOException
* @throws MetricsGraphException
*/
private MetricData getMetricData(String rrdFilename, long startTime, long endTime)
throws IOException, MetricsGraphException {
LOGGER.trace("ENTERING: getMetricData");
// Create RRD DB in read-only mode for the specified RRD file
RrdDb rrdDb = new RrdDb(rrdFilename, true);
// Extract the data source (should always only be one data source per RRD file - otherwise we
// have a problem)
if (rrdDb.getDsCount() != 1) {
throw new MetricsGraphException(
"Only one data source per RRD file is supported - RRD file "
+ rrdFilename
+ " has "
+ rrdDb.getDsCount()
+ " data sources.");
}
// The step (sample) interval that determines how often RRD collects the metric's data
long rrdStep = rrdDb.getRrdDef().getStep();
// Retrieve the RRD file's data source type (COUNTER or GAUGE) to determine how (later)
// to store the metric's data for presentation.
DsType dataSourceType = rrdDb.getDatasource(0).getType();
// Fetch the metric's data from the RRD file for the specified time range
FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.TOTAL, startTime, endTime);
FetchData fetchData = fetchRequest.fetchData();
long[] timestamps = fetchData.getTimestamps();
double[] values = fetchData.getValues(0);
// Done retrieving data from the RRD database - close it, otherwise no one else will
// be able to access it later.
rrdDb.close();
// The lists of the metric's timestamps and their associated values that have non-"NaN" values
List<Long> validTimestamps = new ArrayList<Long>();
List<Double> validValues = new ArrayList<Double>();
long totalCount = 0;
MetricData metricData = new MetricData();
if (dataSourceType == DsType.COUNTER) {
// Counters are for constantly incrementing data, hence they can
// have a summation of their totals
metricData.setHasTotalCount(true);
for (int i = 0; i < timestamps.length; i++) {
// Filter out the RRD values that have not yet been sampled (they will
// have been set to NaN as a placeholder when the RRD file was created)
if (!Double.toString(values[i]).equals("NaN")) {
// RRD averages the collected samples over the step interval.
// To "undo" this averaging and get the actual count, need to
// multiply the sampled data value by the RRD step interval.
double nonAveragedValue = (double) (values[i] * rrdStep);
validTimestamps.add(timestamps[i]);
validValues.add(nonAveragedValue);
totalCount += (long) nonAveragedValue;
}
}
} else if (dataSourceType == DsType.GAUGE) {
// Gauges are for data that waxes and wanes, hence no total count
metricData.setHasTotalCount(false);
for (int i = 0; i < timestamps.length; i++) {
// Filter out the RRD values that have not yet been sampled (they will
// have been set to NaN as a placeholder when the RRD file was created)
if (!Double.toString(values[i]).equals("NaN")) {
validTimestamps.add(timestamps[i]);
validValues.add(values[i]);
}
}
}
metricData.setTimestamps(validTimestamps);
metricData.setValues(validValues);
metricData.setTotalCount(totalCount);
LOGGER.trace("EXITING: getMetricData");
return metricData;
}
@Override
public byte[] createGraph(
String metricName,
String rrdFilename,
long startTime,
long endTime,
String verticalAxisLabel,
String title)
throws IOException, MetricsGraphException {
// Create RRD DB in read-only mode for the specified RRD file
RrdDb rrdDb = new RrdDb(rrdFilename, true);
// Extract the data source (should always only be one data source per RRD file - otherwise we
// have a problem)
if (rrdDb.getDsCount() != 1) {
throw new MetricsGraphException(
"Only one data source per RRD file is supported - RRD file "
+ rrdFilename
+ " has "
+ rrdDb.getDsCount()
+ " data sources.");
}
// Define attributes of the graph to be created for this metric
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setTimeSpan(startTime, endTime);
graphDef.setImageFormat("PNG");
graphDef.setShowSignature(false);
graphDef.setStep(60);
graphDef.setVerticalLabel(verticalAxisLabel);
graphDef.setHeight(500);
graphDef.setWidth(1000);
graphDef.setTitle(title);
DsType dataSourceType = rrdDb.getDatasource(0).getType();
// Determine if the Data Source for this RRD file is a COUNTER or GAUGE
// (Need to know this because COUNTER data is averaged across samples and the vertical axis of
// the
// generated graph by default will show data per rrdStep interval)
if (dataSourceType == DsType.COUNTER) {
// If we ever needed to adjust the metric's data collected by RRD by the archive step
// (which is the rrdStep * archiveSampleCount) this is how to do it.
// FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.AVERAGE,
// startTime, endTime);
// Archive archive = rrdDb.findMatchingArchive(fetchRequest);
// long archiveStep = archive.getArcStep();
// LOGGER.debug("archiveStep = " + archiveStep);
long rrdStep = rrdDb.getRrdDef().getStep();
LOGGER.debug("rrdStep = " + rrdStep);
// Still TBD if we want to graph the AVERAGE data on the same graph
// graphDef.comment(metricName + " ");
// graphDef.datasource("myAverage", rrdFilename, "data", ConsolFun.AVERAGE);
// graphDef.datasource("realAverage", "myAverage," + rrdStep + ",*");
// graphDef.line("realAverage", Color.GREEN, "Average", 2);
// Multiplied by the rrdStep to "undo" the automatic averaging that RRD does
// when it collects TOTAL data - we want the actual totals for the step, not
// the average of the totals.
graphDef.datasource("myTotal", rrdFilename, "data", ConsolFun.TOTAL);
graphDef.datasource("realTotal", "myTotal," + rrdStep + ",*");
graphDef.line("realTotal", Color.BLUE, convertCamelCase(metricName), 2);
// Add some spacing between the graph and the summary stats shown beneath the graph
graphDef.comment("\\s");
graphDef.comment("\\s");
graphDef.comment("\\c");
// Average, Min, and Max over all of the TOTAL data - displayed at bottom of the graph
graphDef.gprint("realTotal", ConsolFun.AVERAGE, "Average = %.3f%s");
graphDef.gprint("realTotal", ConsolFun.MIN, "Min = %.3f%s");
graphDef.gprint("realTotal", ConsolFun.MAX, "Max = %.3f%s");
} else if (dataSourceType == DsType.GAUGE) {
graphDef.datasource("myAverage", rrdFilename, "data", ConsolFun.AVERAGE);
graphDef.line("myAverage", Color.RED, convertCamelCase(metricName), 2);
// Add some spacing between the graph and the summary stats shown beneath the graph
graphDef.comment("\\s");
graphDef.comment("\\s");
graphDef.comment("\\c");
// Average, Min, and Max over all of the AVERAGE data - displayed at bottom of the graph
graphDef.gprint("myAverage", ConsolFun.AVERAGE, "Average = %.3f%s");
graphDef.gprint("myAverage", ConsolFun.MIN, "Min = %.3f%s");
graphDef.gprint("myAverage", ConsolFun.MAX, "Max = %.3f%s");
} else {
rrdDb.close();
throw new MetricsGraphException(
"Unsupported data source type "
+ dataSourceType.name()
+ " in RRD file "
+ rrdFilename
+ ", only COUNTER and GAUGE data source types supported.");
}
rrdDb.close();
// Use "-" as filename so that RRD creates the graph only in memory (no file is
// created, hence no file locking problems due to race conditions between multiple clients)
graphDef.setFilename("-");
RrdGraph graph = new RrdGraph(graphDef);
return graph.getRrdGraphInfo().getBytes();
}