private void writeSubSingleBenchmarkScoreCharts() {
subSingleBenchmarkAggregationChartFileMap = new HashMap<ProblemBenchmarkResult, List<File>>();
CategoryAxis xAxis = new CategoryAxis("Solver Configurations");
NumberAxis yAxis = new NumberAxis("Scores distribution of single benchmark runs");
yAxis.setStandardTickUnits(NumberAxis.createIntegerTickUnits());
yAxis.setAutoRangeIncludesZero(false);
BoxAndWhiskerRenderer renderer =
new BoxAndWhiskerRenderer() {
@Override
public int
getRowCount() { // TODO: HACK for https://issues.jboss.org/browse/PLANNER-429 center
// plotted boxes to x axis labels
return 1;
}
};
renderer.setFillBox(true);
renderer.setUseOutlinePaintForWhiskers(true);
renderer.setMedianVisible(true);
renderer.setMeanVisible(false);
renderer.setItemMargin(0.0);
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
List<? extends BoxAndWhiskerCategoryDataset> datasetList =
generateSubSingleBenchmarkScoreSummary(problemBenchmarkResult);
List<File> chartFileList = new ArrayList<File>(datasetList.size());
int scoreLevelIndex = 0;
for (BoxAndWhiskerCategoryDataset dataset : datasetList) {
CategoryPlot plot = new CategoryPlot(dataset, xAxis, yAxis, renderer);
plot.setOrientation(PlotOrientation.VERTICAL);
JFreeChart chart =
new JFreeChart(
problemBenchmarkResult
+ " (level "
+ scoreLevelIndex
+ ") single benchmark runs score distribution",
JFreeChart.DEFAULT_TITLE_FONT,
plot,
true);
chartFileList.add(
writeChartToImageFile(
chart,
"SubSingleSummary"
+ problemBenchmarkResult.getAnchorId()
+ "Level"
+ scoreLevelIndex));
scoreLevelIndex++;
}
subSingleBenchmarkAggregationChartFileMap.put(problemBenchmarkResult, chartFileList);
}
}
private void determineDefaultShownScoreLevelIndex() {
defaultShownScoreLevelIndex = Integer.MAX_VALUE;
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
if (problemBenchmarkResult.hasAnySuccess()) {
double[] winningScoreLevels =
ScoreUtils.extractLevelDoubles(
problemBenchmarkResult.getWinningSingleBenchmarkResult().getAverageScore());
int[] differenceCount = new int[winningScoreLevels.length];
for (int i = 0; i < differenceCount.length; i++) {
differenceCount[i] = 0;
}
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
double[] scoreLevels =
ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getAverageScore());
for (int i = 0; i < scoreLevels.length; i++) {
if (scoreLevels[i] != winningScoreLevels[i]) {
differenceCount[i] = differenceCount[i] + 1;
}
}
}
}
int firstInterestingLevel = differenceCount.length - 1;
for (int i = 0; i < differenceCount.length; i++) {
if (differenceCount[i] > 0) {
firstInterestingLevel = i;
break;
}
}
if (defaultShownScoreLevelIndex > firstInterestingLevel) {
defaultShownScoreLevelIndex = firstInterestingLevel;
}
}
}
}
private List<? extends BoxAndWhiskerCategoryDataset> generateSubSingleBenchmarkScoreSummary(
ProblemBenchmarkResult problemBenchmarkResult) {
List<DefaultBoxAndWhiskerCategoryDataset> datasetList =
new ArrayList<DefaultBoxAndWhiskerCategoryDataset>(CHARTED_SCORE_LEVEL_SIZE);
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
List<List<Double>> valueListList = new ArrayList<List<Double>>(CHARTED_SCORE_LEVEL_SIZE);
for (SubSingleBenchmarkResult subSingleBenchmarkResult :
singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
if (subSingleBenchmarkResult.hasAllSuccess() && subSingleBenchmarkResult.isInitialized()) {
double[] levelValues =
ScoreUtils.extractLevelDoubles(subSingleBenchmarkResult.getAverageScore());
for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
if (i >= valueListList.size()) {
valueListList.add(new ArrayList<Double>(singleBenchmarkResult.getSuccessCount()));
}
valueListList.get(i).add(levelValues[i]);
}
}
}
for (int i = 0; i < valueListList.size() && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
if (i >= datasetList.size()) {
datasetList.add(new DefaultBoxAndWhiskerCategoryDataset());
}
SolverBenchmarkResult solverBenchmarkResult =
singleBenchmarkResult.getSolverBenchmarkResult();
datasetList
.get(i)
.add(
valueListList.get(i),
solverBenchmarkResult.getName(),
solverBenchmarkResult
+ " - "
+ solverBenchmarkResult.getSubSingleCount()
+ " run(s)");
}
}
return datasetList;
}
public void writeReport() {
logger.info("Generating benchmark report...");
summaryDirectory = new File(plannerBenchmarkResult.getBenchmarkReportDirectory(), "summary");
summaryDirectory.mkdir();
plannerBenchmarkResult.accumulateResults(this);
fillWarningList();
writeBestScoreSummaryCharts();
writeBestScoreScalabilitySummaryChart();
writeWinningScoreDifferenceSummaryChart();
writeWorstScoreDifferencePercentageSummaryChart();
writeAverageCalculateCountPerSecondSummaryChart();
writeTimeSpentSummaryChart();
writeTimeSpentScalabilitySummaryChart();
writeBestScorePerTimeSpentSummaryChart();
writeSubSingleBenchmarkScoreCharts();
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
for (SubSingleBenchmarkResult subSingleBenchmarkResult :
singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
if (!subSingleBenchmarkResult.hasAllSuccess()) {
continue;
}
for (SubSingleStatistic subSingleStatistic :
subSingleBenchmarkResult.getEffectiveSubSingleStatisticMap().values()) {
try {
subSingleStatistic.unhibernatePointList();
} catch (IllegalStateException e) {
if (!plannerBenchmarkResult.getAggregation()) {
throw new IllegalStateException(
"Failed to unhibernate point list of SubSingleStatistic ("
+ subSingleStatistic
+ ") of SubSingleBenchmark ("
+ subSingleBenchmarkResult
+ ").",
e);
}
logger.trace(
"This is expected, aggregator doesn't copy CSV files. Could not read CSV file "
+ "({}) of sub single statistic ({}).",
subSingleStatistic.getCsvFile().getAbsolutePath(),
subSingleStatistic);
}
}
}
}
}
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
if (problemBenchmarkResult.hasAnySuccess()) {
for (ProblemStatistic problemStatistic : problemBenchmarkResult.getProblemStatisticList()) {
problemStatistic.writeGraphFiles(this);
}
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
for (PureSubSingleStatistic pureSubSingleStatistic :
singleBenchmarkResult.getMedian().getPureSubSingleStatisticList()) {
pureSubSingleStatistic.writeGraphFiles(this);
}
}
}
}
}
for (ProblemBenchmarkResult problemBenchmarkResult :
plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
for (SingleBenchmarkResult singleBenchmarkResult :
problemBenchmarkResult.getSingleBenchmarkResultList()) {
for (SubSingleBenchmarkResult subSingleBenchmarkResult :
singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
if (!subSingleBenchmarkResult.hasAllSuccess()) {
continue;
}
for (SubSingleStatistic subSingleStatistic :
subSingleBenchmarkResult.getEffectiveSubSingleStatisticMap().values()) {
if (plannerBenchmarkResult.getAggregation()) {
subSingleStatistic.setPointList(null);
} else {
subSingleStatistic.hibernatePointList();
}
}
}
}
}
determineDefaultShownScoreLevelIndex();
writeHtmlOverviewFile();
}