private void writeAverageCalculateCountPerSecondSummaryChart() {
   List<XYSeries> seriesList =
       new ArrayList<XYSeries>(plannerBenchmarkResult.getSolverBenchmarkResultList().size());
   for (SolverBenchmarkResult solverBenchmarkResult :
       plannerBenchmarkResult.getSolverBenchmarkResultList()) {
     String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
     XYSeries series = new XYSeries(solverLabel);
     for (SingleBenchmarkResult singleBenchmarkResult :
         solverBenchmarkResult.getSingleBenchmarkResultList()) {
       if (singleBenchmarkResult.hasAllSuccess()) {
         long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
         long averageCalculateCountPerSecond =
             singleBenchmarkResult.getAverageCalculateCountPerSecond();
         series.add((Long) problemScale, (Long) averageCalculateCountPerSecond);
       }
     }
     seriesList.add(series);
   }
   XYPlot plot =
       createScalabilityPlot(
           seriesList,
           "Problem scale",
           NumberFormat.getInstance(locale),
           "Average calculate count per second",
           NumberFormat.getInstance(locale));
   JFreeChart chart =
       new JFreeChart(
           "Average calculate count summary (higher is better)",
           JFreeChart.DEFAULT_TITLE_FONT,
           plot,
           true);
   averageCalculateCountSummaryChartFile =
       writeChartToImageFile(chart, "averageCalculateCountSummary");
 }
 private void writeTimeSpentScalabilitySummaryChart() {
   List<XYSeries> seriesList =
       new ArrayList<XYSeries>(plannerBenchmarkResult.getSolverBenchmarkResultList().size());
   for (SolverBenchmarkResult solverBenchmarkResult :
       plannerBenchmarkResult.getSolverBenchmarkResultList()) {
     String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
     XYSeries series = new XYSeries(solverLabel);
     for (SingleBenchmarkResult singleBenchmarkResult :
         solverBenchmarkResult.getSingleBenchmarkResultList()) {
       if (singleBenchmarkResult.hasAllSuccess()) {
         long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
         long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
         series.add((Long) problemScale, (Long) timeMillisSpent);
       }
     }
     seriesList.add(series);
   }
   XYPlot plot =
       createScalabilityPlot(
           seriesList,
           "Problem scale",
           NumberFormat.getInstance(locale),
           "Time spent",
           new MillisecondsSpentNumberFormat(locale));
   JFreeChart chart =
       new JFreeChart(
           "Time spent scalability summary (lower is better)",
           JFreeChart.DEFAULT_TITLE_FONT,
           plot,
           true);
   timeSpentScalabilitySummaryChartFile =
       writeChartToImageFile(chart, "timeSpentScalabilitySummary");
 }
 private void writeBestScorePerTimeSpentSummaryChart() {
   // Each scoreLevel has it's own dataset and chartFile
   List<List<XYSeries>> seriesListList = new ArrayList<List<XYSeries>>(CHARTED_SCORE_LEVEL_SIZE);
   int solverBenchmarkIndex = 0;
   for (SolverBenchmarkResult solverBenchmarkResult :
       plannerBenchmarkResult.getSolverBenchmarkResultList()) {
     String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
     for (SingleBenchmarkResult singleBenchmarkResult :
         solverBenchmarkResult.getSingleBenchmarkResultList()) {
       if (singleBenchmarkResult.hasAllSuccess()) {
         long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
         double[] levelValues =
             ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getAverageScore());
         for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
           if (i >= seriesListList.size()) {
             seriesListList.add(
                 new ArrayList<XYSeries>(
                     plannerBenchmarkResult.getSolverBenchmarkResultList().size()));
           }
           List<XYSeries> seriesList = seriesListList.get(i);
           while (solverBenchmarkIndex >= seriesList.size()) {
             seriesList.add(new XYSeries(solverLabel));
           }
           seriesList
               .get(solverBenchmarkIndex)
               .add((Long) timeMillisSpent, (Double) levelValues[i]);
         }
       }
     }
     solverBenchmarkIndex++;
   }
   bestScorePerTimeSpentSummaryChartFileList = new ArrayList<File>(seriesListList.size());
   int scoreLevelIndex = 0;
   for (List<XYSeries> seriesList : seriesListList) {
     XYPlot plot =
         createScalabilityPlot(
             seriesList,
             "Time spent",
             new MillisecondsSpentNumberFormat(locale),
             "Score level " + scoreLevelIndex,
             NumberFormat.getInstance(locale));
     JFreeChart chart =
         new JFreeChart(
             "Best score per time spent level "
                 + scoreLevelIndex
                 + " summary (higher left is better)",
             JFreeChart.DEFAULT_TITLE_FONT,
             plot,
             true);
     bestScorePerTimeSpentSummaryChartFileList.add(
         writeChartToImageFile(chart, "bestScorePerTimeSpentSummaryLevel" + scoreLevelIndex));
     scoreLevelIndex++;
   }
 }
  protected static SingleBenchmarkResult createMerge(
      SolverBenchmarkResult solverBenchmarkResult,
      ProblemBenchmarkResult problemBenchmarkResult,
      SingleBenchmarkResult oldResult) {
    SingleBenchmarkResult newResult =
        new SingleBenchmarkResult(solverBenchmarkResult, problemBenchmarkResult);
    newResult.pureSingleStatisticList =
        new ArrayList<PureSingleStatistic>(oldResult.pureSingleStatisticList.size());
    for (PureSingleStatistic oldSingleStatistic : oldResult.pureSingleStatisticList) {
      newResult.pureSingleStatisticList.add(
          oldSingleStatistic.getStatisticType().buildPureSingleStatistic(newResult));
    }

    newResult.initSingleStatisticMap();
    for (SingleStatistic singleStatistic : newResult.effectiveSingleStatisticMap.values()) {
      singleStatistic.setPointList(
          oldResult.getSingleStatistic(singleStatistic.getStatisticType()).getPointList());
    }
    // Skip oldResult.reportDirectory
    // Skip oldResult.usedMemoryAfterInputSolution
    newResult.succeeded = oldResult.succeeded;
    newResult.score = oldResult.score;
    newResult.timeMillisSpent = oldResult.timeMillisSpent;
    newResult.calculateCount = oldResult.calculateCount;

    solverBenchmarkResult.getSingleBenchmarkResultList().add(newResult);
    problemBenchmarkResult.getSingleBenchmarkResultList().add(newResult);
    return newResult;
  }
 private void writeWorstScoreDifferencePercentageSummaryChart() {
   // Each scoreLevel has it's own dataset and chartFile
   List<DefaultCategoryDataset> datasetList =
       new ArrayList<DefaultCategoryDataset>(CHARTED_SCORE_LEVEL_SIZE);
   for (SolverBenchmarkResult solverBenchmarkResult :
       plannerBenchmarkResult.getSolverBenchmarkResultList()) {
     String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
     for (SingleBenchmarkResult singleBenchmarkResult :
         solverBenchmarkResult.getSingleBenchmarkResultList()) {
       String planningProblemLabel = singleBenchmarkResult.getProblemBenchmarkResult().getName();
       if (singleBenchmarkResult.hasAllSuccess()) {
         double[] levelValues =
             singleBenchmarkResult.getWorstScoreDifferencePercentage().getPercentageLevels();
         for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
           if (i >= datasetList.size()) {
             datasetList.add(new DefaultCategoryDataset());
           }
           datasetList.get(i).addValue(levelValues[i], solverLabel, planningProblemLabel);
         }
       }
     }
   }
   worstScoreDifferencePercentageSummaryChartFileList = new ArrayList<File>(datasetList.size());
   int scoreLevelIndex = 0;
   for (DefaultCategoryDataset dataset : datasetList) {
     CategoryPlot plot =
         createBarChartPlot(
             dataset,
             "Worst score difference percentage level " + scoreLevelIndex,
             NumberFormat.getPercentInstance(locale));
     JFreeChart chart =
         new JFreeChart(
             "Worst score difference percentage level "
                 + scoreLevelIndex
                 + " summary (higher is better)",
             JFreeChart.DEFAULT_TITLE_FONT,
             plot,
             true);
     worstScoreDifferencePercentageSummaryChartFileList.add(
         writeChartToImageFile(
             chart, "worstScoreDifferencePercentageSummaryLevel" + scoreLevelIndex));
     scoreLevelIndex++;
   }
 }
 private void writeTimeSpentSummaryChart() {
   DefaultCategoryDataset dataset = new DefaultCategoryDataset();
   for (SolverBenchmarkResult solverBenchmarkResult :
       plannerBenchmarkResult.getSolverBenchmarkResultList()) {
     String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
     for (SingleBenchmarkResult singleBenchmarkResult :
         solverBenchmarkResult.getSingleBenchmarkResultList()) {
       String planningProblemLabel = singleBenchmarkResult.getProblemBenchmarkResult().getName();
       if (singleBenchmarkResult.hasAllSuccess()) {
         long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
         dataset.addValue(timeMillisSpent, solverLabel, planningProblemLabel);
       }
     }
   }
   CategoryPlot plot =
       createBarChartPlot(dataset, "Time spent", new MillisecondsSpentNumberFormat(locale));
   JFreeChart chart =
       new JFreeChart(
           "Time spent summary (lower time is better)", JFreeChart.DEFAULT_TITLE_FONT, plot, true);
   timeSpentSummaryChartFile = writeChartToImageFile(chart, "timeSpentSummary");
 }
 private List<? extends BoxAndWhiskerCategoryDataset> generateSubSingleBenchmarkScoreSummary(
     ProblemBenchmarkResult problemBenchmarkResult) {
   List<DefaultBoxAndWhiskerCategoryDataset> datasetList =
       new ArrayList<DefaultBoxAndWhiskerCategoryDataset>(CHARTED_SCORE_LEVEL_SIZE);
   for (SingleBenchmarkResult singleBenchmarkResult :
       problemBenchmarkResult.getSingleBenchmarkResultList()) {
     List<List<Double>> valueListList = new ArrayList<List<Double>>(CHARTED_SCORE_LEVEL_SIZE);
     for (SubSingleBenchmarkResult subSingleBenchmarkResult :
         singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
       if (subSingleBenchmarkResult.hasAllSuccess() && subSingleBenchmarkResult.isInitialized()) {
         double[] levelValues =
             ScoreUtils.extractLevelDoubles(subSingleBenchmarkResult.getAverageScore());
         for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
           if (i >= valueListList.size()) {
             valueListList.add(new ArrayList<Double>(singleBenchmarkResult.getSuccessCount()));
           }
           valueListList.get(i).add(levelValues[i]);
         }
       }
     }
     for (int i = 0; i < valueListList.size() && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
       if (i >= datasetList.size()) {
         datasetList.add(new DefaultBoxAndWhiskerCategoryDataset());
       }
       SolverBenchmarkResult solverBenchmarkResult =
           singleBenchmarkResult.getSolverBenchmarkResult();
       datasetList
           .get(i)
           .add(
               valueListList.get(i),
               solverBenchmarkResult.getName(),
               solverBenchmarkResult
                   + " - "
                   + solverBenchmarkResult.getSubSingleCount()
                   + " run(s)");
     }
   }
   return datasetList;
 }
 private void determineDefaultShownScoreLevelIndex() {
   defaultShownScoreLevelIndex = Integer.MAX_VALUE;
   for (ProblemBenchmarkResult problemBenchmarkResult :
       plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
     if (problemBenchmarkResult.hasAnySuccess()) {
       double[] winningScoreLevels =
           ScoreUtils.extractLevelDoubles(
               problemBenchmarkResult.getWinningSingleBenchmarkResult().getAverageScore());
       int[] differenceCount = new int[winningScoreLevels.length];
       for (int i = 0; i < differenceCount.length; i++) {
         differenceCount[i] = 0;
       }
       for (SingleBenchmarkResult singleBenchmarkResult :
           problemBenchmarkResult.getSingleBenchmarkResultList()) {
         if (singleBenchmarkResult.hasAllSuccess()) {
           double[] scoreLevels =
               ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getAverageScore());
           for (int i = 0; i < scoreLevels.length; i++) {
             if (scoreLevels[i] != winningScoreLevels[i]) {
               differenceCount[i] = differenceCount[i] + 1;
             }
           }
         }
       }
       int firstInterestingLevel = differenceCount.length - 1;
       for (int i = 0; i < differenceCount.length; i++) {
         if (differenceCount[i] > 0) {
           firstInterestingLevel = i;
           break;
         }
       }
       if (defaultShownScoreLevelIndex > firstInterestingLevel) {
         defaultShownScoreLevelIndex = firstInterestingLevel;
       }
     }
   }
 }
 public void writeReport() {
   logger.info("Generating benchmark report...");
   summaryDirectory = new File(plannerBenchmarkResult.getBenchmarkReportDirectory(), "summary");
   summaryDirectory.mkdir();
   plannerBenchmarkResult.accumulateResults(this);
   fillWarningList();
   writeBestScoreSummaryCharts();
   writeBestScoreScalabilitySummaryChart();
   writeWinningScoreDifferenceSummaryChart();
   writeWorstScoreDifferencePercentageSummaryChart();
   writeAverageCalculateCountPerSecondSummaryChart();
   writeTimeSpentSummaryChart();
   writeTimeSpentScalabilitySummaryChart();
   writeBestScorePerTimeSpentSummaryChart();
   writeSubSingleBenchmarkScoreCharts();
   for (ProblemBenchmarkResult problemBenchmarkResult :
       plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
     for (SingleBenchmarkResult singleBenchmarkResult :
         problemBenchmarkResult.getSingleBenchmarkResultList()) {
       for (SubSingleBenchmarkResult subSingleBenchmarkResult :
           singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
         if (!subSingleBenchmarkResult.hasAllSuccess()) {
           continue;
         }
         for (SubSingleStatistic subSingleStatistic :
             subSingleBenchmarkResult.getEffectiveSubSingleStatisticMap().values()) {
           try {
             subSingleStatistic.unhibernatePointList();
           } catch (IllegalStateException e) {
             if (!plannerBenchmarkResult.getAggregation()) {
               throw new IllegalStateException(
                   "Failed to unhibernate point list of SubSingleStatistic ("
                       + subSingleStatistic
                       + ") of SubSingleBenchmark ("
                       + subSingleBenchmarkResult
                       + ").",
                   e);
             }
             logger.trace(
                 "This is expected, aggregator doesn't copy CSV files. Could not read CSV file "
                     + "({}) of sub single statistic ({}).",
                 subSingleStatistic.getCsvFile().getAbsolutePath(),
                 subSingleStatistic);
           }
         }
       }
     }
   }
   for (ProblemBenchmarkResult problemBenchmarkResult :
       plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
     if (problemBenchmarkResult.hasAnySuccess()) {
       for (ProblemStatistic problemStatistic : problemBenchmarkResult.getProblemStatisticList()) {
         problemStatistic.writeGraphFiles(this);
       }
       for (SingleBenchmarkResult singleBenchmarkResult :
           problemBenchmarkResult.getSingleBenchmarkResultList()) {
         if (singleBenchmarkResult.hasAllSuccess()) {
           for (PureSubSingleStatistic pureSubSingleStatistic :
               singleBenchmarkResult.getMedian().getPureSubSingleStatisticList()) {
             pureSubSingleStatistic.writeGraphFiles(this);
           }
         }
       }
     }
   }
   for (ProblemBenchmarkResult problemBenchmarkResult :
       plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
     for (SingleBenchmarkResult singleBenchmarkResult :
         problemBenchmarkResult.getSingleBenchmarkResultList()) {
       for (SubSingleBenchmarkResult subSingleBenchmarkResult :
           singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
         if (!subSingleBenchmarkResult.hasAllSuccess()) {
           continue;
         }
         for (SubSingleStatistic subSingleStatistic :
             subSingleBenchmarkResult.getEffectiveSubSingleStatisticMap().values()) {
           if (plannerBenchmarkResult.getAggregation()) {
             subSingleStatistic.setPointList(null);
           } else {
             subSingleStatistic.hibernatePointList();
           }
         }
       }
     }
   }
   determineDefaultShownScoreLevelIndex();
   writeHtmlOverviewFile();
 }
 @Override
 public void writeGraphFiles(BenchmarkReport benchmarkReport) {
   List<XYPlot> plotList = new ArrayList<XYPlot>(BenchmarkReport.CHARTED_SCORE_LEVEL_SIZE);
   int seriesIndex = 0;
   for (SingleBenchmarkResult singleBenchmarkResult :
       problemBenchmarkResult.getSingleBenchmarkResultList()) {
     List<XYSeries> seriesList = new ArrayList<XYSeries>(BenchmarkReport.CHARTED_SCORE_LEVEL_SIZE);
     // No direct ascending lines between 2 points, but a stepping line instead
     XYItemRenderer renderer = new XYStepRenderer();
     if (singleBenchmarkResult.isSuccess()) {
       BestScoreSingleStatistic singleStatistic =
           (BestScoreSingleStatistic)
               singleBenchmarkResult.getSingleStatistic(problemStatisticType);
       for (BestScoreStatisticPoint point : singleStatistic.getPointList()) {
         long timeMillisSpent = point.getTimeMillisSpent();
         double[] levelValues = ScoreUtils.extractLevelDoubles(point.getScore());
         for (int i = 0;
             i < levelValues.length && i < BenchmarkReport.CHARTED_SCORE_LEVEL_SIZE;
             i++) {
           if (i >= seriesList.size()) {
             seriesList.add(
                 new XYSeries(
                     singleBenchmarkResult
                         .getSolverBenchmarkResult()
                         .getNameWithFavoriteSuffix()));
           }
           seriesList.get(i).add(timeMillisSpent, levelValues[i]);
         }
       }
       // TODO if startingSolution is initialized and no improvement is made, a horizontal line
       // should be shown
       // Draw a horizontal line from the last new best step to how long the solver actually ran
       long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
       double[] bestScoreLevels = ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getScore());
       for (int i = 0;
           i < bestScoreLevels.length && i < BenchmarkReport.CHARTED_SCORE_LEVEL_SIZE;
           i++) {
         if (i >= seriesList.size()) {
           seriesList.add(
               new XYSeries(
                   singleBenchmarkResult.getSolverBenchmarkResult().getNameWithFavoriteSuffix()));
         }
         seriesList.get(i).add(timeMillisSpent, bestScoreLevels[i]);
       }
       if (singleStatistic.getPointList().size() <= 1) {
         // Workaround for
         // https://sourceforge.net/tracker/?func=detail&aid=3387330&group_id=15494&atid=115494
         renderer = new StandardXYItemRenderer(StandardXYItemRenderer.SHAPES_AND_LINES);
       }
     }
     if (singleBenchmarkResult.getSolverBenchmarkResult().isFavorite()) {
       // Make the favorite more obvious
       renderer.setSeriesStroke(0, new BasicStroke(2.0f));
     }
     for (int i = 0; i < seriesList.size(); i++) {
       if (i >= plotList.size()) {
         plotList.add(createPlot(benchmarkReport, i));
       }
       plotList.get(i).setDataset(seriesIndex, new XYSeriesCollection(seriesList.get(i)));
       plotList.get(i).setRenderer(seriesIndex, renderer);
     }
     seriesIndex++;
   }
   graphFileList = new ArrayList<File>(plotList.size());
   for (int scoreLevelIndex = 0; scoreLevelIndex < plotList.size(); scoreLevelIndex++) {
     JFreeChart chart =
         new JFreeChart(
             problemBenchmarkResult.getName()
                 + " best score level "
                 + scoreLevelIndex
                 + " statistic",
             JFreeChart.DEFAULT_TITLE_FONT,
             plotList.get(scoreLevelIndex),
             true);
     graphFileList.add(
         writeChartToImageFile(
             chart,
             problemBenchmarkResult.getName() + "BestScoreStatisticLevel" + scoreLevelIndex));
   }
 }