private void parseXGrid(String xgrid) {
if (xgrid == null) {
return;
}
if (xgrid.equalsIgnoreCase("none")) {
gdef.setDrawXGrid(false);
return;
}
String[] tokens = new ColonSplitter(xgrid).split();
if (tokens.length != 8) {
throw new IllegalArgumentException("Invalid XGRID settings: " + xgrid);
}
int minorUnit = resolveUnit(tokens[0]),
majorUnit = resolveUnit(tokens[2]),
labelUnit = resolveUnit(tokens[4]);
int minorUnitCount = parseInt(tokens[1]),
majorUnitCount = parseInt(tokens[3]),
labelUnitCount = parseInt(tokens[5]);
int labelSpan = parseInt(tokens[6]);
String fmt = tokens[7];
gdef.setTimeAxis(
minorUnit,
minorUnitCount,
majorUnit,
majorUnitCount,
labelUnit,
labelUnitCount,
labelSpan,
fmt);
}
private void datasource(String name, String rpn) {
if (graphDef != null) {
graphDef.datasource(name, rpn);
} else {
dp.addDatasource(name, rpn);
}
}
private void parseComment(String word) {
String[] tokens = new ColonSplitter(word).split();
if (tokens.length != 2) {
throw new IllegalArgumentException("Invalid COMMENT specification: " + word);
}
gdef.comment(tokens[1]);
}
private void percentile(String name, String dsName, int percentile) {
if (graphDef != null) {
graphDef.datasource(name, dsName, new Variable.PERCENTILE(percentile));
} else {
dp.addDatasource(name, dsName, new Variable.PERCENTILE(percentile));
}
}
private void datasource(String name, Plottable plottable) {
if (graphDef != null) {
graphDef.datasource(name, plottable);
} else {
dp.addDatasource(name, plottable);
}
}
private void parseGPrint(String word) {
String[] tokens = new ColonSplitter(word).split();
if (tokens.length != 4) {
throw new IllegalArgumentException("Invalid GPRINT specification: " + word);
}
gdef.gprint(tokens[1], ConsolFun.valueOf(tokens[2]), tokens[3]);
}
private void parseYGrid(String ygrid) {
if (ygrid == null) {
return;
}
if (ygrid.equalsIgnoreCase("none")) {
gdef.setDrawYGrid(false);
return;
}
String[] tokens = new ColonSplitter(ygrid).split();
if (tokens.length != 2) {
throw new IllegalArgumentException("Invalid YGRID settings: " + ygrid);
}
double gridStep = parseDouble(tokens[0]);
int labelFactor = parseInt(tokens[1]);
gdef.setValueAxis(gridStep, labelFactor);
}
private void parseCDef(String word) {
String[] tokens1 = new ColonSplitter(word).split();
if (tokens1.length != 2) {
throw new IllegalArgumentException("Invalid CDEF specification: " + word);
}
String[] tokens2 = tokens1[1].split("=");
if (tokens2.length != 2) {
throw new IllegalArgumentException("Invalid DEF specification: " + word);
}
gdef.datasource(tokens2[0], tokens2[1]);
}
private void parseHRule(String word) {
String[] tokens1 = new ColonSplitter(word).split();
if (tokens1.length < 2 || tokens1.length > 3) {
throw new IllegalArgumentException("Invalid HRULE statement: " + word);
}
String[] tokens2 = tokens1[1].split("#");
if (tokens2.length != 2) {
throw new IllegalArgumentException("Invalid HRULE statement: " + word);
}
double value = parseDouble(tokens2[0]);
Paint color = Util.parseColor(tokens2[1]);
gdef.hrule(value, color, tokens1.length == 3 ? tokens1[2] : null);
}
private void parseVRule(String word) {
String[] tokens1 = new ColonSplitter(word).split();
if (tokens1.length < 2 || tokens1.length > 3) {
throw new IllegalArgumentException("Invalid VRULE statement: " + word);
}
String[] tokens2 = tokens1[1].split("#");
if (tokens2.length != 2) {
throw new IllegalArgumentException("Invalid VRULE statement: " + word);
}
long timestamp = Util.getTimestamp(tokens2[0]);
Paint color = Util.parseColor(tokens2[1]);
gdef.vrule(timestamp, color, tokens1.length == 3 ? tokens1[2] : null);
}
private void parseStack(String word) {
String[] tokens1 = new ColonSplitter(word).split();
if (tokens1.length != 2 && tokens1.length != 3) {
throw new IllegalArgumentException("Invalid STACK statement: " + word);
}
String[] tokens2 = tokens1[1].split("#");
if (tokens2.length != 1 && tokens2.length != 2) {
throw new IllegalArgumentException("Invalid STACK statement: " + word);
}
String name = tokens2[0];
Paint color = tokens2.length == 2 ? Util.parseColor(tokens2[1]) : BLIND_COLOR;
String legend = tokens1.length == 3 ? tokens1[2] : null;
gdef.stack(name, color, legend);
}
private void parseColors(String[] colorOptions) {
if (colorOptions == null) {
return;
}
for (String colorOption : colorOptions) {
String[] tokens = colorOption.split("#");
if (tokens.length != 2) {
throw new IllegalArgumentException("Invalid COLOR specification: " + colorOption);
}
String colorName = tokens[0];
Paint paint = Util.parseColor(tokens[1]);
gdef.setColor(colorName, paint);
}
}
private void parseLine(String word) {
String[] tokens1 = new ColonSplitter(word).split();
if (tokens1.length != 2 && tokens1.length != 3) {
throw new IllegalArgumentException("Invalid LINE statement: " + word);
}
String[] tokens2 = tokens1[1].split("#");
if (tokens2.length != 1 && tokens2.length != 2) {
throw new IllegalArgumentException("Invalid LINE statement: " + word);
}
float width = Integer.parseInt(tokens1[0].substring(tokens1[0].length() - 1));
String name = tokens2[0];
Paint color = tokens2.length == 2 ? Util.parseColor(tokens2[1]) : BLIND_COLOR;
String legend = tokens1.length == 3 ? tokens1[2] : null;
gdef.line(name, color, legend, width);
}
private static byte[] render(RrdGraphDef graphDef) throws IOException {
graphDef.setImageFormat("PNG");
RrdGraph graph = new RrdGraph(graphDef);
BufferedImage bi =
new BufferedImage(
graph.getRrdGraphInfo().getWidth(),
graph.getRrdGraphInfo().getHeight(),
BufferedImage.TYPE_INT_RGB);
graph.render(bi.getGraphics());
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ImageIO.write(bi, "png", bos);
return bos.toByteArray();
}
@Override
public byte[] renderUserMethodGraphForUser(long theUserPid, TimeRange theRange)
throws IOException {
ourLog.info("Rendering user method graph for user {}", theUserPid);
myBroadcastSender.requestFlushQueuedStats();
PersUser user = myDao.getUser(theUserPid);
Date start = theRange.getNoPresetFrom();
Date end = theRange.getNoPresetTo();
if (theRange.getWithPresetRange() != null) {
start =
new Date(
System.currentTimeMillis()
- (theRange.getWithPresetRange().getNumMins() * DateUtils.MILLIS_PER_MINUTE));
end = new Date();
}
HashMap<Long, List<Double>> methods = new HashMap<>();
List<Long> timestamps = new ArrayList<>();
InvocationStatsIntervalEnum nextInterval =
doWithStatsSupportFindInterval(myConfig.getConfig(), start);
Date nextDate = nextInterval.truncate(start);
Date nextDateEnd = null;
List<PersInvocationMethodUserStats> stats = myDao.getUserStatsWithinTimeRange(user, start, end);
Validate.notNull(stats);
stats = new ArrayList<>(stats);
Collections.sort(
stats,
new Comparator<PersInvocationMethodUserStats>() {
@Override
public int compare(
PersInvocationMethodUserStats theO1, PersInvocationMethodUserStats theO2) {
return theO1.getPk().getStartTime().compareTo(theO2.getPk().getStartTime());
}
});
ourLog.debug(
"Loaded {} user {} stats for range {} - {}",
new Object[] {stats.size(), theUserPid, start, end});
if (stats.size() > 0) {
ourLog.debug(
"Found stats with range {} - {}",
stats.get(0).getPk().getStartTime(),
stats.get(stats.size() - 1).getPk().getStartTime());
}
ourLog.debug("Looking for stats starting at {}", nextDate);
Iterator<PersInvocationMethodUserStats> statIter = stats.iterator();
PersInvocationMethodUserStats nextStat = null;
int timesPassed = 0;
int foundEntries = 0;
double grandTotal = 0;
while (nextDate.after(end) == false) {
double numMinutes = nextInterval.numMinutes();
nextDateEnd = DateUtils.addMinutes(nextDate, (int) nextInterval.numMinutes());
timestamps.add(nextDate.getTime());
int arrayIndex = timesPassed;
timesPassed++;
for (List<Double> next : methods.values()) {
next.add(ZERO_DOUBLE);
}
while (nextStat == null || statIter.hasNext()) {
if (nextStat == null && statIter.hasNext()) {
nextStat = statIter.next();
foundEntries++;
}
if (nextStat != null && nextStat.getPk().getStartTime().before(nextDateEnd)) {
long methodPid = nextStat.getPk().getMethod();
List<Double> newList = methods.get(methodPid);
if (newList == null) {
newList = new ArrayList<>();
for (int i = 0; i < timesPassed; i++) {
newList.add(ZERO_DOUBLE);
}
methods.put(methodPid, newList);
}
// TODO: should we have an option to include fails/faults?
double total = nextStat.getSuccessInvocationCount();
double minuteTotal = total / numMinutes;
grandTotal += minuteTotal;
newList.set(arrayIndex, newList.get(arrayIndex) + minuteTotal);
nextStat = null;
} else {
break;
}
}
nextDate = doWithStatsSupportIncrement(nextDate, nextInterval);
nextInterval = doWithStatsSupportFindInterval(myConfig.getConfig(), nextDate);
nextDate = nextInterval.truncate(nextDate);
}
ourLog.debug("Found {} entries for {} methods", new Object[] {foundEntries, methods.size()});
// Come up with a good order
final Map<Long, PersMethod> pidToMethod = new HashMap<>();
for (long nextPid : methods.keySet()) {
PersMethod method = myDao.getServiceVersionMethodByPid(nextPid);
if (method != null) {
pidToMethod.put(nextPid, method);
} else {
ourLog.debug("Discarding unknown method: {}", nextPid);
}
}
ArrayList<Long> pids = new ArrayList<>(pidToMethod.keySet());
Collections.sort(
pids,
new Comparator<Long>() {
@Override
public int compare(Long theO1, Long theO2) {
PersMethod m1 = pidToMethod.get(theO1);
PersMethod m2 = pidToMethod.get(theO2);
return MethodComparator.INSTANCE.compare(m1, m2);
}
});
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setWidth(600);
graphDef.setHeight(200);
long[] graphTimestamps = new long[timestamps.size()];
for (int i = 0; i < timestamps.size(); i++) {
graphTimestamps[i] = timestamps.get(i) / 1000;
}
graphDef.setTimeSpan(graphTimestamps);
graphDef.setVerticalLabel("Calls / Minute");
graphDef.setTextAntiAliasing(true);
int longestName = 0;
for (PersMethod next : pidToMethod.values()) {
longestName = Math.max(longestName, next.getName().length());
}
// Draw the methods
String previousServiceDesc = null;
List<Color> colours = createStackColours(pids.size());
for (int i = 0; i < pids.size(); i++) {
Long nextPid = pids.get(i);
PersMethod nextMethod = pidToMethod.get(nextPid);
List<Double> values = methods.get(nextPid);
LinearInterpolator avgPlot =
new LinearInterpolator(graphTimestamps, toDoublesFromDoubles(values));
String srcName = "inv" + i;
graphDef.datasource(srcName, avgPlot);
String methodDesc =
nextMethod.getServiceVersion().getService().getServiceId()
+ " "
+ nextMethod.getServiceVersion().getVersionId();
if (!StringUtils.equals(previousServiceDesc, methodDesc)) {
graphDef.comment(methodDesc + "\\l");
previousServiceDesc = methodDesc;
}
double sumDoubles = sumDoubles(values);
double pct = (sumDoubles / grandTotal);
if (i == 0) {
graphDef.area(
srcName, colours.get(i), " " + StringUtils.rightPad(nextMethod.getName(), longestName));
} else {
graphDef.stack(
srcName, colours.get(i), " " + StringUtils.rightPad(nextMethod.getName(), longestName));
}
graphDef.gprint(srcName, ConsolFun.AVERAGE, "Avg %5.1f ");
graphDef.gprint(srcName, ConsolFun.MIN, "Min %5.1f ");
graphDef.gprint(srcName, ConsolFun.MAX, "Max %5.1f ");
String formattedPct = new DecimalFormat("0.0#%").format(pct);
graphDef.comment("Pct: " + formattedPct + "\\l");
}
if (pids.size() == 0) {
double[] values = new double[timestamps.size()];
for (int j = 0; j < values.length; j++) {
values[j] = 0.0;
}
LinearInterpolator avgPlot = new LinearInterpolator(graphTimestamps, values);
String srcName = "inv";
graphDef.datasource(srcName, avgPlot);
graphDef.area(
srcName, Color.BLACK, StringUtils.rightPad("No activity during this range", 100));
}
return render(graphDef);
}
/**
* Fill a GraphDef with values as defined by the graph desc
*
* @param graphDef the GraphDef to configure
* @param defProbe The probe to get values from
* @param customData some custom data, they override existing values in the associated probe
*/
public void fillGraphDef(
RrdGraphDef graphDef, Probe<?, ?> defProbe, Map<String, ? extends Plottable> customData) {
HostsList hl = defProbe.getHostList();
List<DsDesc> toDo = new ArrayList<DsDesc>();
// The datasources already found
Set<String> datasources = new HashSet<String>();
for (DsDesc ds : allds) {
boolean complete = false;
// not a data source, don't try to add it in datasources
if (!ds.graphType.datasource()) {
complete = true;
}
// The graph is a percentile
else if (ds.percentile != null) {
complete = true;
graphDef.percentile(ds.name, ds.dsName, ds.percentile);
datasources.add(ds.name);
}
// A rpn datasource
else if (ds.rpn != null) {
complete = true;
if (!datasources.contains(ds.name)) {
graphDef.datasource(ds.name, ds.rpn);
datasources.add(ds.name);
}
} else if (ds.graphType == GraphType.LEGEND) {
complete = true;
}
// Does the datas existe in the provided values
// It override existing values in the probe
else if (customData != null && customData.containsKey(ds.dsName)) {
complete = true;
if (!datasources.contains(ds.name)) {
graphDef.datasource(ds.name, customData.get(ds.dsName));
datasources.add(ds.name);
logger.trace(Util.delayedFormatString("custom data found for %s", ds.dsName));
}
}
// Last but common case, datasource refers to a rrd
// Or they might be on the associated rrd
else {
Probe<?, ?> probe = defProbe;
if (ds.dspath != null) {
if (logger.isTraceEnabled())
logger.trace(
"External probe path: " + ds.dspath.host + "/" + ds.dspath.probe + "/" + ds.dsName);
probe = hl.getProbeByPath(ds.dspath.host, ds.dspath.probe);
if (probe == null) {
logger.error("Invalide probe: " + ds.dspath.host + "/" + ds.dspath.probe);
continue;
}
}
if (!probe.dsExist(ds.dsName)) {
logger.error("Invalide datasource " + ds.dsName + ", not found in " + probe);
continue;
}
complete = true;
if (!datasources.contains(ds.name)) {
String rrdName = probe.getRrdName();
graphDef.datasource(ds.name, rrdName, ds.dsName, ds.cf);
datasources.add(ds.name);
} else {
logger.error(
"Datasource '" + ds.name + "' defined twice in " + name + ", for found: " + ds);
}
}
if (complete) {
toDo.add(ds);
} else {
logger.debug("Error for " + ds);
logger.error("No way to plot " + ds.name + " in " + name + " found");
}
}
// The title line, only if values block is required
if (withSummary) {
graphDef.comment(""); // We simulate the color box
graphDef.comment(MANYSPACE.substring(0, Math.min(maxLengthLegend, MANYSPACE.length()) + 2));
graphDef.comment("Current");
graphDef.comment(" Average");
graphDef.comment(" Minimum");
graphDef.comment(" Maximum");
graphDef.comment("\\l");
}
if (logger.isTraceEnabled()) {
logger.trace("Datasource: " + datasources);
logger.trace("Todo: " + toDo);
}
String shortLegend = withSummary ? " \\g" : null;
for (DsDesc ds : toDo) {
ds.graphType.draw(graphDef, ds.name, ds.color, shortLegend);
if (withSummary && ds.graphType.legend())
addLegend(graphDef, ds.name, ds.graphType, ds.legend);
}
}
public RrdGraphDef getEmptyGraphDef() {
RrdGraphDef retValue = new RrdGraphDef();
if (!Double.isNaN(lowerLimit)) retValue.setMinValue(lowerLimit);
if (!Double.isNaN(upperLimit)) retValue.setMaxValue(upperLimit);
if (verticalLabel != null) retValue.setVerticalLabel(verticalLabel);
if (this.siUnit) retValue.setBase(1000);
else retValue.setBase(1024);
if (unitExponent != null) {
retValue.setUnitsExponent(unitExponent);
}
retValue.setLogarithmic(logarithmic);
retValue.setPoolUsed(true);
retValue.setAntiAliasing(true);
retValue.setTextAntiAliasing(true);
retValue.setImageFormat("PNG");
retValue.setWidth(getWidth());
retValue.setHeight(getHeight());
return retValue;
}
@Override
public byte[] renderSvcVerLatencyMethodGraph(
long theSvcVerPid, TimeRange theRange, boolean theIndividualMethod) throws IOException {
ourLog.info("Rendering user method graph for Service Version {}", theSvcVerPid);
myBroadcastSender.requestFlushQueuedStats();
BasePersServiceVersion svcVer = myServiceRegistry.getServiceVersionByPid(theSvcVerPid);
/*
* Init the graph
*/
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setWidth(600);
graphDef.setHeight(200);
graphDef.setTitle(
"Backing Service Latency: "
+ svcVer.getService().getDomain().getDomainId()
+ " / "
+ svcVer.getService().getServiceId()
+ " / "
+ svcVer.getVersionId());
graphDef.setVerticalLabel("Milliseconds / Call");
graphDef.setTextAntiAliasing(true);
graphDef.setMinValue(0.0);
/*
* Loop through each method and load the latency stats
*/
final List<String> names = new ArrayList<>();
final List<List<Long>> latencyLists = new ArrayList<>();
final List<Long> timestamps = new ArrayList<>();
for (PersMethod nextMethod : svcVer.getMethods()) {
if (BaseDtoServiceVersion.METHOD_NAME_UNKNOWN.equals(nextMethod.getName())) {
continue;
}
final List<Long> latencyMin;
if (theIndividualMethod) {
names.add(nextMethod.getName());
latencyMin = new ArrayList<>();
latencyLists.add(latencyMin);
} else {
if (names.isEmpty()) {
names.add("All Methods");
latencyMin = new ArrayList<>();
latencyLists.add(latencyMin);
} else {
latencyMin = latencyLists.get(0);
}
}
doWithStatsByMinute(
myConfig.getConfig(),
theRange,
myStatus,
nextMethod,
new IWithStats<PersInvocationMethodSvcverStatsPk, PersInvocationMethodSvcverStats>() {
@Override
public void withStats(int theIndex, PersInvocationMethodSvcverStats theStats) {
growToSizeLong(latencyMin, theIndex);
growToSizeLong(timestamps, theIndex);
long latency =
theStats.getSuccessInvocationTotalTime() > 0
? theStats.getSuccessInvocationTotalTime()
/ theStats.getSuccessInvocationCount()
: 0;
latencyMin.set(theIndex, addToLong(latencyMin.get(theIndex), latency));
timestamps.set(theIndex, theStats.getPk().getStartTime().getTime());
}
});
}
/*
* Set time span
*/
long[] graphTimestamps = new long[timestamps.size()];
for (int i = 0; i < timestamps.size(); i++) {
graphTimestamps[i] = timestamps.get(i) / 1000;
}
graphDef.setTimeSpan(graphTimestamps);
/*
* Figure out the longest name
*/
int longestName = 0;
for (String next : names) {
longestName = Math.max(longestName, next.length());
}
/*
* Straighten
*/
int numWithValues = 0;
List<Boolean> hasValuesList = new ArrayList<>();
for (List<Long> nextList : latencyLists) {
boolean hasValues = false;
for (int i = 0; i < nextList.size(); i++) {
long l = nextList.get(i);
if (l == 0) {
if (i > 0 && nextList.get(i - 1) > 0) {
nextList.set(i, nextList.get(i - 1));
}
} else {
hasValues = true;
}
}
hasValuesList.add(hasValues);
if (hasValues) {
numWithValues++;
}
}
/*
* Figure out colours
*/
List<Color> colours = new ArrayList<>();
int colourIndex = 0;
for (int i = 0; i < hasValuesList.size(); i++) {
if (hasValuesList.get(i)) {
colours.add(createStackColour(numWithValues, colourIndex++));
} else {
colours.add(Color.black);
}
}
/*
* Add the lines to the graph
*/
for (int i = 0; i < names.size(); i++) {
String name = names.get(i);
List<Long> latencyMin = latencyLists.get(i);
Plottable avgPlot = new LinearInterpolator(graphTimestamps, toDoublesFromLongs(latencyMin));
String srcName = "inv" + i;
graphDef.datasource(srcName, avgPlot);
graphDef.line(srcName, colours.get(i), " " + StringUtils.rightPad(name, longestName), 2);
if (hasValuesList.get(i)) {
graphDef.gprint(srcName, ConsolFun.AVERAGE, "Avg %5.1f ");
graphDef.gprint(srcName, ConsolFun.MIN, "Min %5.1f ");
graphDef.gprint(srcName, ConsolFun.MAX, "Max %5.1f \\l");
} else {
graphDef.comment("No Invocations During This Period\\l");
}
}
return render(graphDef);
}
@Override
public byte[] createGraph(
String metricName,
String rrdFilename,
long startTime,
long endTime,
String verticalAxisLabel,
String title)
throws IOException, MetricsGraphException {
// Create RRD DB in read-only mode for the specified RRD file
RrdDb rrdDb = new RrdDb(rrdFilename, true);
// Extract the data source (should always only be one data source per RRD file - otherwise we
// have a problem)
if (rrdDb.getDsCount() != 1) {
throw new MetricsGraphException(
"Only one data source per RRD file is supported - RRD file "
+ rrdFilename
+ " has "
+ rrdDb.getDsCount()
+ " data sources.");
}
// Define attributes of the graph to be created for this metric
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setTimeSpan(startTime, endTime);
graphDef.setImageFormat("PNG");
graphDef.setShowSignature(false);
graphDef.setStep(60);
graphDef.setVerticalLabel(verticalAxisLabel);
graphDef.setHeight(500);
graphDef.setWidth(1000);
graphDef.setTitle(title);
DsType dataSourceType = rrdDb.getDatasource(0).getType();
// Determine if the Data Source for this RRD file is a COUNTER or GAUGE
// (Need to know this because COUNTER data is averaged across samples and the vertical axis of
// the
// generated graph by default will show data per rrdStep interval)
if (dataSourceType == DsType.COUNTER) {
// If we ever needed to adjust the metric's data collected by RRD by the archive step
// (which is the rrdStep * archiveSampleCount) this is how to do it.
// FetchRequest fetchRequest = rrdDb.createFetchRequest(ConsolFun.AVERAGE,
// startTime, endTime);
// Archive archive = rrdDb.findMatchingArchive(fetchRequest);
// long archiveStep = archive.getArcStep();
// LOGGER.debug("archiveStep = " + archiveStep);
long rrdStep = rrdDb.getRrdDef().getStep();
LOGGER.debug("rrdStep = " + rrdStep);
// Still TBD if we want to graph the AVERAGE data on the same graph
// graphDef.comment(metricName + " ");
// graphDef.datasource("myAverage", rrdFilename, "data", ConsolFun.AVERAGE);
// graphDef.datasource("realAverage", "myAverage," + rrdStep + ",*");
// graphDef.line("realAverage", Color.GREEN, "Average", 2);
// Multiplied by the rrdStep to "undo" the automatic averaging that RRD does
// when it collects TOTAL data - we want the actual totals for the step, not
// the average of the totals.
graphDef.datasource("myTotal", rrdFilename, "data", ConsolFun.TOTAL);
graphDef.datasource("realTotal", "myTotal," + rrdStep + ",*");
graphDef.line("realTotal", Color.BLUE, convertCamelCase(metricName), 2);
// Add some spacing between the graph and the summary stats shown beneath the graph
graphDef.comment("\\s");
graphDef.comment("\\s");
graphDef.comment("\\c");
// Average, Min, and Max over all of the TOTAL data - displayed at bottom of the graph
graphDef.gprint("realTotal", ConsolFun.AVERAGE, "Average = %.3f%s");
graphDef.gprint("realTotal", ConsolFun.MIN, "Min = %.3f%s");
graphDef.gprint("realTotal", ConsolFun.MAX, "Max = %.3f%s");
} else if (dataSourceType == DsType.GAUGE) {
graphDef.datasource("myAverage", rrdFilename, "data", ConsolFun.AVERAGE);
graphDef.line("myAverage", Color.RED, convertCamelCase(metricName), 2);
// Add some spacing between the graph and the summary stats shown beneath the graph
graphDef.comment("\\s");
graphDef.comment("\\s");
graphDef.comment("\\c");
// Average, Min, and Max over all of the AVERAGE data - displayed at bottom of the graph
graphDef.gprint("myAverage", ConsolFun.AVERAGE, "Average = %.3f%s");
graphDef.gprint("myAverage", ConsolFun.MIN, "Min = %.3f%s");
graphDef.gprint("myAverage", ConsolFun.MAX, "Max = %.3f%s");
} else {
rrdDb.close();
throw new MetricsGraphException(
"Unsupported data source type "
+ dataSourceType.name()
+ " in RRD file "
+ rrdFilename
+ ", only COUNTER and GAUGE data source types supported.");
}
rrdDb.close();
// Use "-" as filename so that RRD creates the graph only in memory (no file is
// created, hence no file locking problems due to race conditions between multiple clients)
graphDef.setFilename("-");
RrdGraph graph = new RrdGraph(graphDef);
return graph.getRrdGraphInfo().getBytes();
}
@Override
public byte[] renderSvcVerPayloadSizeGraph(long theServiceVersionPid, TimeRange theRange)
throws IOException {
ourLog.info("Rendering payload size graph for service version {}", theServiceVersionPid);
myBroadcastSender.requestFlushQueuedStats();
final List<Integer> invCount = new ArrayList<>();
final List<Long> totalSuccessReqBytes = new ArrayList<>();
final List<Long> totalSuccessRespBytes = new ArrayList<>();
final List<Long> timestamps = new ArrayList<>();
BasePersServiceVersion svcVer = myServiceRegistry.getServiceVersionByPid(theServiceVersionPid);
for (PersMethod nextMethod : svcVer.getMethods()) {
doWithStatsByMinute(
myConfig.getConfig(),
theRange,
myStatus,
nextMethod,
new IWithStats<PersInvocationMethodSvcverStatsPk, PersInvocationMethodSvcverStats>() {
@Override
public void withStats(int theIndex, PersInvocationMethodSvcverStats theStats) {
growToSizeInt(invCount, theIndex);
growToSizeLong(totalSuccessRespBytes, theIndex);
growToSizeLong(totalSuccessReqBytes, theIndex);
growToSizeLong(timestamps, theIndex);
totalSuccessReqBytes.set(
theIndex,
addToLong(
totalSuccessReqBytes.get(theIndex),
theStats.getSuccessRequestMessageBytes()));
totalSuccessRespBytes.set(
theIndex,
addToLong(
totalSuccessRespBytes.get(theIndex),
theStats.getSuccessResponseMessageBytes()));
invCount.set(
theIndex, addToInt(invCount.get(theIndex), theStats.getSuccessInvocationCount()));
timestamps.set(theIndex, theStats.getPk().getStartTime().getTime());
}
});
}
double[] avgSuccessReqSize = new double[invCount.size()];
double[] avgSuccessRespSize = new double[invCount.size()];
for (int i = 0; i < invCount.size(); i++) {
avgSuccessReqSize[i] =
invCount.get(i) == 0 ? 0 : totalSuccessReqBytes.get(i) / invCount.get(i);
avgSuccessRespSize[i] =
invCount.get(i) == 0 ? 0 : totalSuccessRespBytes.get(i) / invCount.get(i);
}
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setWidth(600);
graphDef.setHeight(200);
graphDef.setTitle(
"Message Payload Size: "
+ svcVer.getService().getDomain().getDomainId()
+ " / "
+ svcVer.getService().getServiceId()
+ " / "
+ svcVer.getVersionId());
long[] timestamps1 = new long[invCount.size()];
double prevReq = 0.0;
double prevResp = 0.0;
for (int i = 0; i < invCount.size(); i++) {
timestamps1[i] = timestamps.get(i) / 1000;
prevReq = avgSuccessReqSize[i] > 0 ? avgSuccessReqSize[i] : prevReq;
avgSuccessReqSize[i] = prevReq;
prevResp = avgSuccessRespSize[i] > 0 ? avgSuccessRespSize[i] : prevResp;
avgSuccessRespSize[i] = prevResp;
}
for (int i = 0; i < avgSuccessReqSize.length; i++) {}
graphDef.setTimeSpan(timestamps1);
graphDef.setVerticalLabel("Bytes / Message");
graphDef.setTextAntiAliasing(true);
LinearInterpolator reqPlot = new LinearInterpolator(timestamps1, avgSuccessReqSize);
graphDef.datasource("req", reqPlot);
graphDef.line("req", Color.RED, "Requests ", 2.0f);
graphDef.gprint("req", ConsolFun.AVERAGE, "Average Size %8.1f ");
graphDef.gprint("req", ConsolFun.MIN, "Min %8.1f ");
graphDef.gprint("req", ConsolFun.MAX, "Max %8.1f\\l");
LinearInterpolator respPlot = new LinearInterpolator(timestamps1, avgSuccessRespSize);
graphDef.datasource("resp", respPlot);
graphDef.line("resp", Color.GREEN, "Responses ", 2.0f);
graphDef.gprint("resp", ConsolFun.AVERAGE, "Average Size %8.1f ");
graphDef.gprint("resp", ConsolFun.MIN, "Min %8.1f ");
graphDef.gprint("resp", ConsolFun.MAX, "Max %8.1f\\l");
return render(graphDef);
}
@Override
public byte[] renderSvcVerThrottlingGraph(long theServiceVersionPid, TimeRange theRange)
throws IOException {
ourLog.info("Rendering throttling graph for service version {}", theServiceVersionPid);
myBroadcastSender.requestFlushQueuedStats();
final List<Long> throttleAcceptCount = new ArrayList<>();
final List<Long> throttleRejectCount = new ArrayList<>();
final List<Long> timestamps = new ArrayList<>();
BasePersServiceVersion svcVer = myServiceRegistry.getServiceVersionByPid(theServiceVersionPid);
for (PersMethod nextMethod : svcVer.getMethods()) {
doWithStatsByMinute(
myConfig.getConfig(),
theRange,
myStatus,
nextMethod,
new IWithStats<PersInvocationMethodSvcverStatsPk, PersInvocationMethodSvcverStats>() {
@Override
public void withStats(int theIndex, PersInvocationMethodSvcverStats theStats) {
growToSizeLong(throttleAcceptCount, theIndex);
growToSizeLong(throttleRejectCount, theIndex);
growToSizeLong(timestamps, theIndex);
double numMinutes = theStats.getPk().getInterval().numMinutes();
throttleAcceptCount.set(
theIndex,
throttleAcceptCount.get(theIndex)
+ (int) (theStats.getTotalThrottleAccepts() / numMinutes));
throttleRejectCount.set(
theIndex,
throttleRejectCount.get(theIndex)
+ (int) (theStats.getTotalThrottleRejections() / numMinutes));
timestamps.set(theIndex, theStats.getPk().getStartTime().getTime());
}
});
}
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setWidth(600);
graphDef.setHeight(200);
graphDef.setTitle(
"Request Throttling: "
+ svcVer.getService().getDomain().getDomainId()
+ " / "
+ svcVer.getService().getServiceId()
+ " / "
+ svcVer.getVersionId());
long[] timestamps1 = new long[timestamps.size()];
for (int i = 0; i < timestamps.size(); i++) {
timestamps1[i] = timestamps.get(i) / 1000;
}
graphDef.setTimeSpan(timestamps1);
graphDef.setVerticalLabel("Throttled Calls / Min");
graphDef.setTextAntiAliasing(true);
LinearInterpolator avgPlot =
new LinearInterpolator(timestamps1, toDoublesFromLongs(throttleAcceptCount));
graphDef.datasource("inv", avgPlot);
graphDef.area(
"inv", Color.GREEN, "Accepted (Throttled but call was allowed to proceed after delay)\\l");
graphDef.gprint("inv", ConsolFun.AVERAGE, "Average %8.1f ");
graphDef.gprint("inv", ConsolFun.MIN, "Min %8.1f ");
graphDef.gprint("inv", ConsolFun.MAX, "Max %8.1f\\l");
LinearInterpolator avgFaultPlot =
new LinearInterpolator(timestamps1, toDoublesFromLongs(throttleRejectCount));
graphDef.datasource("invfault", avgFaultPlot);
graphDef.stack("invfault", Color.BLUE, "Rejected (Throttle queue was full or not allowed)\\l");
graphDef.gprint("invfault", ConsolFun.AVERAGE, "Average %8.1f ");
graphDef.gprint("invfault", ConsolFun.MIN, "Min %8.1f ");
graphDef.gprint("invfault", ConsolFun.MAX, "Max %8.1f\\l");
return render(graphDef);
}
@Override
public byte[] renderSvcVerUsageGraph(long theServiceVersionPid, TimeRange theRange)
throws IOException {
ourLog.info("Rendering latency graph for service version {}", theServiceVersionPid);
myBroadcastSender.requestFlushQueuedStats();
final List<Double> invCount = new ArrayList<>();
final List<Double> invCountFault = new ArrayList<>();
final List<Double> invCountFail = new ArrayList<>();
final List<Double> invCountSecurityFail = new ArrayList<>();
final List<Long> timestamps = new ArrayList<>();
BasePersServiceVersion svcVer = myServiceRegistry.getServiceVersionByPid(theServiceVersionPid);
for (PersMethod nextMethod : svcVer.getMethods()) {
doWithStatsByMinute(
myConfig.getConfig(),
theRange,
myStatus,
nextMethod,
new IWithStats<PersInvocationMethodSvcverStatsPk, PersInvocationMethodSvcverStats>() {
@Override
public void withStats(int theIndex, PersInvocationMethodSvcverStats theStats) {
growToSizeDouble(invCount, theIndex);
growToSizeDouble(invCountFault, theIndex);
growToSizeDouble(invCountFail, theIndex);
growToSizeDouble(invCountSecurityFail, theIndex);
growToSizeLong(timestamps, theIndex);
double numMinutes = theStats.getPk().getInterval().numMinutes();
invCount.set(
theIndex,
invCount.get(theIndex) + (theStats.getSuccessInvocationCount() / numMinutes));
invCountFault.set(
theIndex,
invCountFault.get(theIndex) + (theStats.getFaultInvocationCount() / numMinutes));
invCountFail.set(
theIndex,
invCountFail.get(theIndex) + (theStats.getFailInvocationCount() / numMinutes));
invCountSecurityFail.set(
theIndex,
invCountSecurityFail.get(theIndex)
+ (theStats.getServerSecurityFailures() / numMinutes));
timestamps.set(theIndex, theStats.getPk().getStartTime().getTime());
}
});
}
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setWidth(600);
graphDef.setHeight(200);
graphDef.setTitle(
"Usage: "
+ svcVer.getService().getDomain().getDomainId()
+ " / "
+ svcVer.getService().getServiceId()
+ " / "
+ svcVer.getVersionId());
long[] timestamps1 = new long[invCount.size()];
for (int i = 0; i < invCount.size(); i++) {
timestamps1[i] = timestamps.get(i) / 1000;
}
graphDef.setTimeSpan(timestamps1);
graphDef.setVerticalLabel("Calls / Min");
graphDef.setTextAntiAliasing(true);
LinearInterpolator avgPlot =
new LinearInterpolator(timestamps1, toDoublesFromDoubles(invCount));
graphDef.datasource("inv", avgPlot);
if (hasValues(invCount)) {
graphDef.area("inv", Color.decode("#00C000"), StringUtils.rightPad("Successful Calls", 20));
graphDef.gprint("inv", ConsolFun.AVERAGE, "Average %8.1f ");
graphDef.gprint("inv", ConsolFun.MIN, "Min %8.1f ");
graphDef.gprint("inv", ConsolFun.MAX, "Max %8.1f\\l");
} else {
graphDef.area(
"inv",
Color.decode("#00C000"),
StringUtils.rightPad("No Successful calls during this time period\\l", 20));
}
if (hasValues(invCountFault)) {
LinearInterpolator avgFaultPlot =
new LinearInterpolator(timestamps1, toDoublesFromDoubles(invCountFault));
graphDef.datasource("invfault", avgFaultPlot);
graphDef.stack("invfault", Color.decode("#6060C0"), StringUtils.rightPad("Faults", 20));
graphDef.gprint("invfault", ConsolFun.AVERAGE, "Average %8.1f ");
graphDef.gprint("invfault", ConsolFun.MIN, "Min %8.1f ");
graphDef.gprint("invfault", ConsolFun.MAX, "Max %8.1f\\l");
} else {
graphDef.comment("No Faults during this time period\\l");
}
if (hasValues(invCountFail)) {
LinearInterpolator avgFailPlot =
new LinearInterpolator(timestamps1, toDoublesFromDoubles(invCountFail));
graphDef.datasource("invfail", avgFailPlot);
graphDef.stack("invfail", Color.decode("#F00000"), StringUtils.rightPad("Fails", 20));
graphDef.gprint("invfail", ConsolFun.AVERAGE, "Average %8.1f ");
graphDef.gprint("invfail", ConsolFun.MIN, "Min %8.1f ");
graphDef.gprint("invfail", ConsolFun.MAX, "Max %8.1f\\l");
} else {
graphDef.comment("No Failures during this time period\\l");
}
if (hasValues(invCountSecurityFail)) {
LinearInterpolator avgSecurityFailPlot =
new LinearInterpolator(timestamps1, toDoublesFromDoubles(invCountSecurityFail));
graphDef.datasource("invSecurityFail", avgSecurityFailPlot);
graphDef.stack(
"invSecurityFail", Color.decode("#F0A000"), StringUtils.rightPad("Security Fails", 20));
graphDef.gprint("invSecurityFail", ConsolFun.AVERAGE, "Average %8.1f ");
graphDef.gprint("invSecurityFail", ConsolFun.MIN, "Min %8.1f ");
graphDef.gprint("invSecurityFail", ConsolFun.MAX, "Max %8.1f\\l");
} else {
graphDef.comment("No Security Failures during this time period\\l");
}
return render(graphDef);
}
public void draw(RrdGraphDef rgd, String sn, Color color, String legend) {
rgd.stack(sn, color, legend);
};
public RrdGraphDef getGraphDef(
long nowInSeconds, SourceIdentifier sourceIdentifier, boolean showMax) {
String absoluteRrdPath = getRrdFile(sourceIdentifier).getAbsolutePath();
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setColor(RrdGraphConstants.COLOR_CANVAS, new Color(0xcc, 0xcc, 0xcc));
graphDef.setNoMinorGrid(true);
graphDef.setShowSignature(false);
graphDef.setMinValue(0);
graphDef.setAltAutoscaleMax(true);
graphDef.setAltYGrid(false);
graphDef.setTimeAxis(
RrdGraphConstants.HOUR,
1,
RrdGraphConstants.HOUR,
6,
RrdGraphConstants.DAY,
1,
0,
"yyyy-MM-dd");
graphDef.setFilename("-");
graphDef.setImageFormat("PNG");
ConsolFun consolFun;
String description;
if (showMax) {
consolFun = ConsolFun.MAX;
description = " (max.)";
} else {
consolFun = ConsolFun.AVERAGE;
description = " (avg.)";
}
graphDef.setVerticalLabel("Events/s" + description);
graphDef.datasource(
RrdLoggingEventHandler.TRACE,
absoluteRrdPath,
RrdLoggingEventHandler.TRACE_DS_NAME,
consolFun);
graphDef.datasource(
RrdLoggingEventHandler.DEBUG,
absoluteRrdPath,
RrdLoggingEventHandler.DEBUG_DS_NAME,
consolFun);
graphDef.datasource(
RrdLoggingEventHandler.INFO,
absoluteRrdPath,
RrdLoggingEventHandler.INFO_DS_NAME,
consolFun);
graphDef.datasource(
RrdLoggingEventHandler.WARN,
absoluteRrdPath,
RrdLoggingEventHandler.WARN_DS_NAME,
consolFun);
graphDef.datasource(
RrdLoggingEventHandler.ERROR,
absoluteRrdPath,
RrdLoggingEventHandler.ERROR_DS_NAME,
consolFun);
graphDef.area(
RrdLoggingEventHandler.TRACE, new Color(0x00, 0x00, 0xff), RrdLoggingEventHandler.TRACE);
graphDef.stack(
RrdLoggingEventHandler.DEBUG, new Color(0x00, 0xff, 0x00), RrdLoggingEventHandler.DEBUG);
graphDef.stack(
RrdLoggingEventHandler.INFO, new Color(0xff, 0xff, 0xff), RrdLoggingEventHandler.INFO);
graphDef.stack(
RrdLoggingEventHandler.WARN, new Color(0xff, 0xff, 0x00), RrdLoggingEventHandler.WARN);
graphDef.stack(
RrdLoggingEventHandler.ERROR, new Color(0xff, 0x00, 0x00), RrdLoggingEventHandler.ERROR);
if (showMax) {
graphDef.datasource(
RrdLoggingEventHandler.TOTAL,
absoluteRrdPath,
RrdLoggingEventHandler.TOTAL_DS_NAME,
consolFun);
graphDef.line(RrdLoggingEventHandler.TOTAL, Color.BLACK, RrdLoggingEventHandler.TOTAL);
}
graphDef.setAntiAliasing(true);
graphDef.setLazy(false);
String sourceTitle = createGraphTitle(sourceIdentifier);
long before = nowInSeconds - 7 * 24 * 60 * 60;
graphDef.setTimeSpan(before, nowInSeconds);
graphDef.setTitle(sourceTitle);
graphDef.setWidth(graphSize.width);
graphDef.setHeight(graphSize.height);
return graphDef;
}
Object execute() throws IOException {
gdef = new RrdGraphDef();
// OPTIONS
// START, END
String t1 = getOptionValue("s", "start", DEFAULT_START),
t2 = getOptionValue("e", "end", DEFAULT_END);
gdef.setTimeSpan(Util.getTimestamps(t1, t2));
// X-GRID
parseXGrid(getOptionValue("x", "x-grid"));
// Y-GRID
parseYGrid(getOptionValue("y", "y-grid"));
// ALT-Y-GRID
gdef.setAltYGrid(getBooleanOption("Y", "alt-y-grid"));
// NO_MINOR
gdef.setNoMinorGrid(getBooleanOption(null, "no-minor"));
// ALT-Y-MRTG
gdef.setAltYMrtg(getBooleanOption("R", "alt-y-mrtg"));
// ALT-AUTOSCALE
gdef.setAltAutoscale(getBooleanOption("A", "alt-autoscale"));
// ALT-AUTOSCALE-MAX
gdef.setAltAutoscaleMax(getBooleanOption("M", "alt-autoscale-max"));
// UNITS-EXPONENT
String opt = getOptionValue("X", "units-exponent");
if (opt != null) {
gdef.setUnitsExponent(parseInt(opt));
}
// UNITS-LENGTH
opt = getOptionValue("L", "units-length");
if (opt != null) {
gdef.setUnitsLength(parseInt(opt));
}
// VERTICAL LABEL
opt = getOptionValue("v", "vertical-label");
if (opt != null) {
gdef.setVerticalLabel(opt);
}
// WIDTH
opt = getOptionValue("w", "width");
if (opt != null) {
gdef.setWidth(parseInt(opt));
}
// HEIGHT
opt = getOptionValue("h", "height");
if (opt != null) {
gdef.setHeight(parseInt(opt));
}
// INTERLACED
gdef.setInterlaced(getBooleanOption("i", "interlaced"));
// IMGINFO
opt = getOptionValue("f", "imginfo");
if (opt != null) {
gdef.setImageInfo(opt);
}
// IMGFORMAT
opt = getOptionValue("a", "imgformat");
if (opt != null) {
gdef.setImageFormat(opt);
}
// BACKGROUND
opt = getOptionValue("B", "background");
if (opt != null) {
gdef.setBackgroundImage(opt);
}
// OVERLAY
opt = getOptionValue("O", "overlay");
if (opt != null) {
gdef.setOverlayImage(opt);
}
// UNIT
opt = getOptionValue("U", "unit");
if (opt != null) {
gdef.setUnit(opt);
}
// LAZY
gdef.setLazy(getBooleanOption("z", "lazy"));
// UPPER-LIMIT
opt = getOptionValue("u", "upper-limit");
if (opt != null) {
gdef.setMaxValue(parseDouble(opt));
}
// LOWER-LIMIT
opt = getOptionValue("l", "lower-limit");
if (opt != null) {
gdef.setMinValue(parseDouble(opt));
}
// RIGID
gdef.setRigid(getBooleanOption("r", "rigid"));
// BASE
opt = getOptionValue("b", "base");
if (opt != null) {
gdef.setBase(parseDouble(opt));
}
// LOGARITHMIC
gdef.setLogarithmic(getBooleanOption("o", "logarithmic"));
// COLORS
parseColors(getMultipleOptionValues("c", "color"));
// NO-LEGEND
gdef.setNoLegend(getBooleanOption("g", "no-legend"));
// ONLY_GRAPH
gdef.setOnlyGraph(getBooleanOption("j", "only-graph"));
// FORCE-RULES-LEGEND
gdef.setForceRulesLegend(getBooleanOption("F", "force-rules-legend"));
// TITLE
opt = getOptionValue("t", "title");
if (opt != null) {
gdef.setTitle(opt);
}
// STEP
opt = getOptionValue("S", "step");
if (opt != null) {
gdef.setStep(parseLong(opt));
}
// NON-OPTIONS
String[] words = getRemainingWords();
// the first word must be a filename
if (words.length < 2) {
throw new IllegalArgumentException("Image filename must be specified");
}
gdef.setFilename(words[1]);
// parse remaining words, in no particular order
for (int i = 2; i < words.length; i++) {
if (words[i].startsWith("DEF:")) {
parseDef(words[i]);
} else if (words[i].startsWith("CDEF:")) {
parseCDef(words[i]);
} else if (words[i].startsWith("PRINT:")) {
parsePrint(words[i]);
} else if (words[i].startsWith("GPRINT:")) {
parseGPrint(words[i]);
} else if (words[i].startsWith("COMMENT:")) {
parseComment(words[i]);
} else if (words[i].startsWith("HRULE:")) {
parseHRule(words[i]);
} else if (words[i].startsWith("VRULE:")) {
parseVRule(words[i]);
} else if (words[i].startsWith("LINE1:")
|| words[i].startsWith("LINE2:")
|| words[i].startsWith("LINE3:")) {
parseLine(words[i]);
} else if (words[i].startsWith("AREA:")) {
parseArea(words[i]);
} else if (words[i].startsWith("STACK:")) {
parseStack(words[i]);
} else {
throw new IllegalArgumentException("Unexpected GRAPH token encountered: " + words[i]);
}
}
// create diagram finally
RrdGraphInfo info = new RrdGraph(gdef).getRrdGraphInfo();
if (info.getFilename().equals(RrdGraphConstants.IN_MEMORY_IMAGE)) {
println(new String(info.getBytes()));
} else {
println(info.getWidth() + "x" + info.getHeight());
String[] plines = info.getPrintLines();
for (String pline : plines) {
println(pline);
}
if (info.getImgInfo() != null && info.getImgInfo().length() > 0) {
println(info.getImgInfo());
}
}
return info;
}
protected void addLegend(RrdGraphDef def, String ds, GraphType gt, String legend) {
if (legend == null) return;
if (gt == GraphType.PERCENTILELEGEND) {
def.comment(legend + "\\g");
int missingLength = Math.min(maxLengthLegend - legend.length(), MANYSPACE.length()) + 2;
if (missingLength > 0) def.comment(MANYSPACE.substring(0, missingLength));
def.gprint(ds, ConsolFun.MAX, "%6.2f%s");
def.comment("\\l");
} else if (gt == GraphType.COMMENT) {
def.comment(legend + "\\l");
} else if (gt != GraphType.NONE) {
def.comment(legend + "\\g");
int missingLength = Math.min(maxLengthLegend - legend.length(), MANYSPACE.length()) + 2;
if (missingLength > 0) def.comment(MANYSPACE.substring(0, missingLength));
def.gprint(ds, ConsolFun.LAST, "%6.2f%s");
def.gprint(ds, ConsolFun.AVERAGE, "%8.2f%s");
def.gprint(ds, ConsolFun.MIN, "%8.2f%s");
def.gprint(ds, ConsolFun.MAX, "%8.2f%s");
def.comment("\\l");
}
}
