private void writeTimeSpentScalabilitySummaryChart() { List<XYSeries> seriesList = new ArrayList<XYSeries>(plannerBenchmarkResult.getSolverBenchmarkResultList().size()); for (SolverBenchmarkResult solverBenchmarkResult : plannerBenchmarkResult.getSolverBenchmarkResultList()) { String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix(); XYSeries series = new XYSeries(solverLabel); for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) { if (singleBenchmarkResult.hasAllSuccess()) { long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale(); long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent(); series.add((Long) problemScale, (Long) timeMillisSpent); } } seriesList.add(series); } XYPlot plot = createScalabilityPlot( seriesList, "Problem scale", NumberFormat.getInstance(locale), "Time spent", new MillisecondsSpentNumberFormat(locale)); JFreeChart chart = new JFreeChart( "Time spent scalability summary (lower is better)", JFreeChart.DEFAULT_TITLE_FONT, plot, true); timeSpentScalabilitySummaryChartFile = writeChartToImageFile(chart, "timeSpentScalabilitySummary"); }
private void writeAverageCalculateCountPerSecondSummaryChart() { List<XYSeries> seriesList = new ArrayList<XYSeries>(plannerBenchmarkResult.getSolverBenchmarkResultList().size()); for (SolverBenchmarkResult solverBenchmarkResult : plannerBenchmarkResult.getSolverBenchmarkResultList()) { String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix(); XYSeries series = new XYSeries(solverLabel); for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) { if (singleBenchmarkResult.hasAllSuccess()) { long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale(); long averageCalculateCountPerSecond = singleBenchmarkResult.getAverageCalculateCountPerSecond(); series.add((Long) problemScale, (Long) averageCalculateCountPerSecond); } } seriesList.add(series); } XYPlot plot = createScalabilityPlot( seriesList, "Problem scale", NumberFormat.getInstance(locale), "Average calculate count per second", NumberFormat.getInstance(locale)); JFreeChart chart = new JFreeChart( "Average calculate count summary (higher is better)", JFreeChart.DEFAULT_TITLE_FONT, plot, true); averageCalculateCountSummaryChartFile = writeChartToImageFile(chart, "averageCalculateCountSummary"); }
private void writeBestScorePerTimeSpentSummaryChart() { // Each scoreLevel has it's own dataset and chartFile List<List<XYSeries>> seriesListList = new ArrayList<List<XYSeries>>(CHARTED_SCORE_LEVEL_SIZE); int solverBenchmarkIndex = 0; for (SolverBenchmarkResult solverBenchmarkResult : plannerBenchmarkResult.getSolverBenchmarkResultList()) { String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix(); for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) { if (singleBenchmarkResult.hasAllSuccess()) { long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent(); double[] levelValues = ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getAverageScore()); for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) { if (i >= seriesListList.size()) { seriesListList.add( new ArrayList<XYSeries>( plannerBenchmarkResult.getSolverBenchmarkResultList().size())); } List<XYSeries> seriesList = seriesListList.get(i); while (solverBenchmarkIndex >= seriesList.size()) { seriesList.add(new XYSeries(solverLabel)); } seriesList .get(solverBenchmarkIndex) .add((Long) timeMillisSpent, (Double) levelValues[i]); } } } solverBenchmarkIndex++; } bestScorePerTimeSpentSummaryChartFileList = new ArrayList<File>(seriesListList.size()); int scoreLevelIndex = 0; for (List<XYSeries> seriesList : seriesListList) { XYPlot plot = createScalabilityPlot( seriesList, "Time spent", new MillisecondsSpentNumberFormat(locale), "Score level " + scoreLevelIndex, NumberFormat.getInstance(locale)); JFreeChart chart = new JFreeChart( "Best score per time spent level " + scoreLevelIndex + " summary (higher left is better)", JFreeChart.DEFAULT_TITLE_FONT, plot, true); bestScorePerTimeSpentSummaryChartFileList.add( writeChartToImageFile(chart, "bestScorePerTimeSpentSummaryLevel" + scoreLevelIndex)); scoreLevelIndex++; } }
private void writeWorstScoreDifferencePercentageSummaryChart() { // Each scoreLevel has it's own dataset and chartFile List<DefaultCategoryDataset> datasetList = new ArrayList<DefaultCategoryDataset>(CHARTED_SCORE_LEVEL_SIZE); for (SolverBenchmarkResult solverBenchmarkResult : plannerBenchmarkResult.getSolverBenchmarkResultList()) { String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix(); for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) { String planningProblemLabel = singleBenchmarkResult.getProblemBenchmarkResult().getName(); if (singleBenchmarkResult.hasAllSuccess()) { double[] levelValues = singleBenchmarkResult.getWorstScoreDifferencePercentage().getPercentageLevels(); for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) { if (i >= datasetList.size()) { datasetList.add(new DefaultCategoryDataset()); } datasetList.get(i).addValue(levelValues[i], solverLabel, planningProblemLabel); } } } } worstScoreDifferencePercentageSummaryChartFileList = new ArrayList<File>(datasetList.size()); int scoreLevelIndex = 0; for (DefaultCategoryDataset dataset : datasetList) { CategoryPlot plot = createBarChartPlot( dataset, "Worst score difference percentage level " + scoreLevelIndex, NumberFormat.getPercentInstance(locale)); JFreeChart chart = new JFreeChart( "Worst score difference percentage level " + scoreLevelIndex + " summary (higher is better)", JFreeChart.DEFAULT_TITLE_FONT, plot, true); worstScoreDifferencePercentageSummaryChartFileList.add( writeChartToImageFile( chart, "worstScoreDifferencePercentageSummaryLevel" + scoreLevelIndex)); scoreLevelIndex++; } }
private void writeTimeSpentSummaryChart() { DefaultCategoryDataset dataset = new DefaultCategoryDataset(); for (SolverBenchmarkResult solverBenchmarkResult : plannerBenchmarkResult.getSolverBenchmarkResultList()) { String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix(); for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) { String planningProblemLabel = singleBenchmarkResult.getProblemBenchmarkResult().getName(); if (singleBenchmarkResult.hasAllSuccess()) { long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent(); dataset.addValue(timeMillisSpent, solverLabel, planningProblemLabel); } } } CategoryPlot plot = createBarChartPlot(dataset, "Time spent", new MillisecondsSpentNumberFormat(locale)); JFreeChart chart = new JFreeChart( "Time spent summary (lower time is better)", JFreeChart.DEFAULT_TITLE_FONT, plot, true); timeSpentSummaryChartFile = writeChartToImageFile(chart, "timeSpentSummary"); }
private List<? extends BoxAndWhiskerCategoryDataset> generateSubSingleBenchmarkScoreSummary( ProblemBenchmarkResult problemBenchmarkResult) { List<DefaultBoxAndWhiskerCategoryDataset> datasetList = new ArrayList<DefaultBoxAndWhiskerCategoryDataset>(CHARTED_SCORE_LEVEL_SIZE); for (SingleBenchmarkResult singleBenchmarkResult : problemBenchmarkResult.getSingleBenchmarkResultList()) { List<List<Double>> valueListList = new ArrayList<List<Double>>(CHARTED_SCORE_LEVEL_SIZE); for (SubSingleBenchmarkResult subSingleBenchmarkResult : singleBenchmarkResult.getSubSingleBenchmarkResultList()) { if (subSingleBenchmarkResult.hasAllSuccess() && subSingleBenchmarkResult.isInitialized()) { double[] levelValues = ScoreUtils.extractLevelDoubles(subSingleBenchmarkResult.getAverageScore()); for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) { if (i >= valueListList.size()) { valueListList.add(new ArrayList<Double>(singleBenchmarkResult.getSuccessCount())); } valueListList.get(i).add(levelValues[i]); } } } for (int i = 0; i < valueListList.size() && i < CHARTED_SCORE_LEVEL_SIZE; i++) { if (i >= datasetList.size()) { datasetList.add(new DefaultBoxAndWhiskerCategoryDataset()); } SolverBenchmarkResult solverBenchmarkResult = singleBenchmarkResult.getSolverBenchmarkResult(); datasetList .get(i) .add( valueListList.get(i), solverBenchmarkResult.getName(), solverBenchmarkResult + " - " + solverBenchmarkResult.getSubSingleCount() + " run(s)"); } } return datasetList; }
public Comparable createRankingWeight( List<SolverBenchmarkResult> solverBenchmarkResultList, SolverBenchmarkResult solverBenchmarkResult) { int betterCount = 0; int equalCount = 0; List<Score> scoreList = solverBenchmarkResult.getScoreList(); for (SolverBenchmarkResult otherSolverBenchmarkResult : solverBenchmarkResultList) { if (otherSolverBenchmarkResult != solverBenchmarkResult) { List<Score> otherScoreList = otherSolverBenchmarkResult.getScoreList(); // TODO the scoreList.size() can differ between SolverBenchmarks for (int i = 0; i < scoreList.size(); i++) { Score score = scoreList.get(i); Score otherScore = otherScoreList.get(i); int scoreComparison = score.compareTo(otherScore); if (scoreComparison > 0) { betterCount++; } else if (scoreComparison == 0) { equalCount++; } } } } return new TotalRankSolverRankingWeight(solverBenchmarkResult, betterCount, equalCount); }