private void writeAverageCalculateCountPerSecondSummaryChart() {
List<XYSeries> seriesList =
new ArrayList<XYSeries>(plannerBenchmarkResult.getSolverBenchmarkResultList().size());
for (SolverBenchmarkResult solverBenchmarkResult :
plannerBenchmarkResult.getSolverBenchmarkResultList()) {
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
XYSeries series = new XYSeries(solverLabel);
for (SingleBenchmarkResult singleBenchmarkResult :
solverBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
long averageCalculateCountPerSecond =
singleBenchmarkResult.getAverageCalculateCountPerSecond();
series.add((Long) problemScale, (Long) averageCalculateCountPerSecond);
}
}
seriesList.add(series);
}
XYPlot plot =
createScalabilityPlot(
seriesList,
"Problem scale",
NumberFormat.getInstance(locale),
"Average calculate count per second",
NumberFormat.getInstance(locale));
JFreeChart chart =
new JFreeChart(
"Average calculate count summary (higher is better)",
JFreeChart.DEFAULT_TITLE_FONT,
plot,
true);
averageCalculateCountSummaryChartFile =
writeChartToImageFile(chart, "averageCalculateCountSummary");
}
private void writeTimeSpentScalabilitySummaryChart() {
List<XYSeries> seriesList =
new ArrayList<XYSeries>(plannerBenchmarkResult.getSolverBenchmarkResultList().size());
for (SolverBenchmarkResult solverBenchmarkResult :
plannerBenchmarkResult.getSolverBenchmarkResultList()) {
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
XYSeries series = new XYSeries(solverLabel);
for (SingleBenchmarkResult singleBenchmarkResult :
solverBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
series.add((Long) problemScale, (Long) timeMillisSpent);
}
}
seriesList.add(series);
}
XYPlot plot =
createScalabilityPlot(
seriesList,
"Problem scale",
NumberFormat.getInstance(locale),
"Time spent",
new MillisecondsSpentNumberFormat(locale));
JFreeChart chart =
new JFreeChart(
"Time spent scalability summary (lower is better)",
JFreeChart.DEFAULT_TITLE_FONT,
plot,
true);
timeSpentScalabilitySummaryChartFile =
writeChartToImageFile(chart, "timeSpentScalabilitySummary");
}
private void writeBestScorePerTimeSpentSummaryChart() {
// Each scoreLevel has it's own dataset and chartFile
List<List<XYSeries>> seriesListList = new ArrayList<List<XYSeries>>(CHARTED_SCORE_LEVEL_SIZE);
int solverBenchmarkIndex = 0;
for (SolverBenchmarkResult solverBenchmarkResult :
plannerBenchmarkResult.getSolverBenchmarkResultList()) {
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
for (SingleBenchmarkResult singleBenchmarkResult :
solverBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
double[] levelValues =
ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getAverageScore());
for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
if (i >= seriesListList.size()) {
seriesListList.add(
new ArrayList<XYSeries>(
plannerBenchmarkResult.getSolverBenchmarkResultList().size()));
}
List<XYSeries> seriesList = seriesListList.get(i);
while (solverBenchmarkIndex >= seriesList.size()) {
seriesList.add(new XYSeries(solverLabel));
}
seriesList
.get(solverBenchmarkIndex)
.add((Long) timeMillisSpent, (Double) levelValues[i]);
}
}
}
solverBenchmarkIndex++;
}
bestScorePerTimeSpentSummaryChartFileList = new ArrayList<File>(seriesListList.size());
int scoreLevelIndex = 0;
for (List<XYSeries> seriesList : seriesListList) {
XYPlot plot =
createScalabilityPlot(
seriesList,
"Time spent",
new MillisecondsSpentNumberFormat(locale),
"Score level " + scoreLevelIndex,
NumberFormat.getInstance(locale));
JFreeChart chart =
new JFreeChart(
"Best score per time spent level "
+ scoreLevelIndex
+ " summary (higher left is better)",
JFreeChart.DEFAULT_TITLE_FONT,
plot,
true);
bestScorePerTimeSpentSummaryChartFileList.add(
writeChartToImageFile(chart, "bestScorePerTimeSpentSummaryLevel" + scoreLevelIndex));
scoreLevelIndex++;
}
}
protected static SingleBenchmarkResult createMerge(
SolverBenchmarkResult solverBenchmarkResult,
ProblemBenchmarkResult problemBenchmarkResult,
SingleBenchmarkResult oldResult) {
SingleBenchmarkResult newResult =
new SingleBenchmarkResult(solverBenchmarkResult, problemBenchmarkResult);
newResult.pureSingleStatisticList =
new ArrayList<PureSingleStatistic>(oldResult.pureSingleStatisticList.size());
for (PureSingleStatistic oldSingleStatistic : oldResult.pureSingleStatisticList) {
newResult.pureSingleStatisticList.add(
oldSingleStatistic.getStatisticType().buildPureSingleStatistic(newResult));
}
newResult.initSingleStatisticMap();
for (SingleStatistic singleStatistic : newResult.effectiveSingleStatisticMap.values()) {
singleStatistic.setPointList(
oldResult.getSingleStatistic(singleStatistic.getStatisticType()).getPointList());
}
// Skip oldResult.reportDirectory
// Skip oldResult.usedMemoryAfterInputSolution
newResult.succeeded = oldResult.succeeded;
newResult.score = oldResult.score;
newResult.timeMillisSpent = oldResult.timeMillisSpent;
newResult.calculateCount = oldResult.calculateCount;
solverBenchmarkResult.getSingleBenchmarkResultList().add(newResult);
problemBenchmarkResult.getSingleBenchmarkResultList().add(newResult);
return newResult;
}
private void writeWorstScoreDifferencePercentageSummaryChart() {
// Each scoreLevel has it's own dataset and chartFile
List<DefaultCategoryDataset> datasetList =
new ArrayList<DefaultCategoryDataset>(CHARTED_SCORE_LEVEL_SIZE);
for (SolverBenchmarkResult solverBenchmarkResult :
plannerBenchmarkResult.getSolverBenchmarkResultList()) {
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
for (SingleBenchmarkResult singleBenchmarkResult :
solverBenchmarkResult.getSingleBenchmarkResultList()) {
String planningProblemLabel = singleBenchmarkResult.getProblemBenchmarkResult().getName();
if (singleBenchmarkResult.hasAllSuccess()) {
double[] levelValues =
singleBenchmarkResult.getWorstScoreDifferencePercentage().getPercentageLevels();
for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
if (i >= datasetList.size()) {
datasetList.add(new DefaultCategoryDataset());
}
datasetList.get(i).addValue(levelValues[i], solverLabel, planningProblemLabel);
}
}
}
}
worstScoreDifferencePercentageSummaryChartFileList = new ArrayList<File>(datasetList.size());
int scoreLevelIndex = 0;
for (DefaultCategoryDataset dataset : datasetList) {
CategoryPlot plot =
createBarChartPlot(
dataset,
"Worst score difference percentage level " + scoreLevelIndex,
NumberFormat.getPercentInstance(locale));
JFreeChart chart =
new JFreeChart(
"Worst score difference percentage level "
+ scoreLevelIndex
+ " summary (higher is better)",
JFreeChart.DEFAULT_TITLE_FONT,
plot,
true);
worstScoreDifferencePercentageSummaryChartFileList.add(
writeChartToImageFile(
chart, "worstScoreDifferencePercentageSummaryLevel" + scoreLevelIndex));
scoreLevelIndex++;
}
}
private void writeTimeSpentSummaryChart() {
DefaultCategoryDataset dataset = new DefaultCategoryDataset();
for (SolverBenchmarkResult solverBenchmarkResult :
plannerBenchmarkResult.getSolverBenchmarkResultList()) {
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
for (SingleBenchmarkResult singleBenchmarkResult :
solverBenchmarkResult.getSingleBenchmarkResultList()) {
String planningProblemLabel = singleBenchmarkResult.getProblemBenchmarkResult().getName();
if (singleBenchmarkResult.hasAllSuccess()) {
long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
dataset.addValue(timeMillisSpent, solverLabel, planningProblemLabel);
}
}
}
CategoryPlot plot =
createBarChartPlot(dataset, "Time spent", new MillisecondsSpentNumberFormat(locale));
JFreeChart chart =
new JFreeChart(
"Time spent summary (lower time is better)", JFreeChart.DEFAULT_TITLE_FONT, plot, true);
timeSpentSummaryChartFile = writeChartToImageFile(chart, "timeSpentSummary");
}
private List<? extends BoxAndWhiskerCategoryDataset> generateSubSingleBenchmarkScoreSummary(
ProblemBenchmarkResult problemBenchmarkResult) {
List<DefaultBoxAndWhiskerCategoryDataset> datasetList =
new ArrayList<DefaultBoxAndWhiskerCategoryDataset>(CHARTED_SCORE_LEVEL_SIZE);
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
List<List<Double>> valueListList = new ArrayList<List<Double>>(CHARTED_SCORE_LEVEL_SIZE);
for (SubSingleBenchmarkResult subSingleBenchmarkResult :
singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
if (subSingleBenchmarkResult.hasAllSuccess() && subSingleBenchmarkResult.isInitialized()) {
double[] levelValues =
ScoreUtils.extractLevelDoubles(subSingleBenchmarkResult.getAverageScore());
for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
if (i >= valueListList.size()) {
valueListList.add(new ArrayList<Double>(singleBenchmarkResult.getSuccessCount()));
}
valueListList.get(i).add(levelValues[i]);
}
}
}
for (int i = 0; i < valueListList.size() && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
if (i >= datasetList.size()) {
datasetList.add(new DefaultBoxAndWhiskerCategoryDataset());
}
SolverBenchmarkResult solverBenchmarkResult =
singleBenchmarkResult.getSolverBenchmarkResult();
datasetList
.get(i)
.add(
valueListList.get(i),
solverBenchmarkResult.getName(),
solverBenchmarkResult
+ " - "
+ solverBenchmarkResult.getSubSingleCount()
+ " run(s)");
}
}
return datasetList;
}
private void determineDefaultShownScoreLevelIndex() {
defaultShownScoreLevelIndex = Integer.MAX_VALUE;
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
if (problemBenchmarkResult.hasAnySuccess()) {
double[] winningScoreLevels =
ScoreUtils.extractLevelDoubles(
problemBenchmarkResult.getWinningSingleBenchmarkResult().getAverageScore());
int[] differenceCount = new int[winningScoreLevels.length];
for (int i = 0; i < differenceCount.length; i++) {
differenceCount[i] = 0;
}
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
double[] scoreLevels =
ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getAverageScore());
for (int i = 0; i < scoreLevels.length; i++) {
if (scoreLevels[i] != winningScoreLevels[i]) {
differenceCount[i] = differenceCount[i] + 1;
}
}
}
}
int firstInterestingLevel = differenceCount.length - 1;
for (int i = 0; i < differenceCount.length; i++) {
if (differenceCount[i] > 0) {
firstInterestingLevel = i;
break;
}
}
if (defaultShownScoreLevelIndex > firstInterestingLevel) {
defaultShownScoreLevelIndex = firstInterestingLevel;
}
}
}
}
public void writeReport() {
logger.info("Generating benchmark report...");
summaryDirectory = new File(plannerBenchmarkResult.getBenchmarkReportDirectory(), "summary");
summaryDirectory.mkdir();
plannerBenchmarkResult.accumulateResults(this);
fillWarningList();
writeBestScoreSummaryCharts();
writeBestScoreScalabilitySummaryChart();
writeWinningScoreDifferenceSummaryChart();
writeWorstScoreDifferencePercentageSummaryChart();
writeAverageCalculateCountPerSecondSummaryChart();
writeTimeSpentSummaryChart();
writeTimeSpentScalabilitySummaryChart();
writeBestScorePerTimeSpentSummaryChart();
writeSubSingleBenchmarkScoreCharts();
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
for (SubSingleBenchmarkResult subSingleBenchmarkResult :
singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
if (!subSingleBenchmarkResult.hasAllSuccess()) {
continue;
}
for (SubSingleStatistic subSingleStatistic :
subSingleBenchmarkResult.getEffectiveSubSingleStatisticMap().values()) {
try {
subSingleStatistic.unhibernatePointList();
} catch (IllegalStateException e) {
if (!plannerBenchmarkResult.getAggregation()) {
throw new IllegalStateException(
"Failed to unhibernate point list of SubSingleStatistic ("
+ subSingleStatistic
+ ") of SubSingleBenchmark ("
+ subSingleBenchmarkResult
+ ").",
e);
}
logger.trace(
"This is expected, aggregator doesn't copy CSV files. Could not read CSV file "
+ "({}) of sub single statistic ({}).",
subSingleStatistic.getCsvFile().getAbsolutePath(),
subSingleStatistic);
}
}
}
}
}
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
if (problemBenchmarkResult.hasAnySuccess()) {
for (ProblemStatistic problemStatistic : problemBenchmarkResult.getProblemStatisticList()) {
problemStatistic.writeGraphFiles(this);
}
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
for (PureSubSingleStatistic pureSubSingleStatistic :
singleBenchmarkResult.getMedian().getPureSubSingleStatisticList()) {
pureSubSingleStatistic.writeGraphFiles(this);
}
}
}
}
}
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
for (SubSingleBenchmarkResult subSingleBenchmarkResult :
singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
if (!subSingleBenchmarkResult.hasAllSuccess()) {
continue;
}
for (SubSingleStatistic subSingleStatistic :
subSingleBenchmarkResult.getEffectiveSubSingleStatisticMap().values()) {
if (plannerBenchmarkResult.getAggregation()) {
subSingleStatistic.setPointList(null);
} else {
subSingleStatistic.hibernatePointList();
}
}
}
}
}
determineDefaultShownScoreLevelIndex();
writeHtmlOverviewFile();
}
@Override
public void writeGraphFiles(BenchmarkReport benchmarkReport) {
List<XYPlot> plotList = new ArrayList<XYPlot>(BenchmarkReport.CHARTED_SCORE_LEVEL_SIZE);
int seriesIndex = 0;
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
List<XYSeries> seriesList = new ArrayList<XYSeries>(BenchmarkReport.CHARTED_SCORE_LEVEL_SIZE);
// No direct ascending lines between 2 points, but a stepping line instead
XYItemRenderer renderer = new XYStepRenderer();
if (singleBenchmarkResult.isSuccess()) {
BestScoreSingleStatistic singleStatistic =
(BestScoreSingleStatistic)
singleBenchmarkResult.getSingleStatistic(problemStatisticType);
for (BestScoreStatisticPoint point : singleStatistic.getPointList()) {
long timeMillisSpent = point.getTimeMillisSpent();
double[] levelValues = ScoreUtils.extractLevelDoubles(point.getScore());
for (int i = 0;
i < levelValues.length && i < BenchmarkReport.CHARTED_SCORE_LEVEL_SIZE;
i++) {
if (i >= seriesList.size()) {
seriesList.add(
new XYSeries(
singleBenchmarkResult
.getSolverBenchmarkResult()
.getNameWithFavoriteSuffix()));
}
seriesList.get(i).add(timeMillisSpent, levelValues[i]);
}
}
// TODO if startingSolution is initialized and no improvement is made, a horizontal line
// should be shown
// Draw a horizontal line from the last new best step to how long the solver actually ran
long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
double[] bestScoreLevels = ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getScore());
for (int i = 0;
i < bestScoreLevels.length && i < BenchmarkReport.CHARTED_SCORE_LEVEL_SIZE;
i++) {
if (i >= seriesList.size()) {
seriesList.add(
new XYSeries(
singleBenchmarkResult.getSolverBenchmarkResult().getNameWithFavoriteSuffix()));
}
seriesList.get(i).add(timeMillisSpent, bestScoreLevels[i]);
}
if (singleStatistic.getPointList().size() <= 1) {
// Workaround for
// https://sourceforge.net/tracker/?func=detail&aid=3387330&group_id=15494&atid=115494
renderer = new StandardXYItemRenderer(StandardXYItemRenderer.SHAPES_AND_LINES);
}
}
if (singleBenchmarkResult.getSolverBenchmarkResult().isFavorite()) {
// Make the favorite more obvious
renderer.setSeriesStroke(0, new BasicStroke(2.0f));
}
for (int i = 0; i < seriesList.size(); i++) {
if (i >= plotList.size()) {
plotList.add(createPlot(benchmarkReport, i));
}
plotList.get(i).setDataset(seriesIndex, new XYSeriesCollection(seriesList.get(i)));
plotList.get(i).setRenderer(seriesIndex, renderer);
}
seriesIndex++;
}
graphFileList = new ArrayList<File>(plotList.size());
for (int scoreLevelIndex = 0; scoreLevelIndex < plotList.size(); scoreLevelIndex++) {
JFreeChart chart =
new JFreeChart(
problemBenchmarkResult.getName()
+ " best score level "
+ scoreLevelIndex
+ " statistic",
JFreeChart.DEFAULT_TITLE_FONT,
plotList.get(scoreLevelIndex),
true);
graphFileList.add(
writeChartToImageFile(
chart,
problemBenchmarkResult.getName() + "BestScoreStatisticLevel" + scoreLevelIndex));
}
}
