Exemplo n.º 1
0
  private void writeSubSingleBenchmarkScoreCharts() {
    subSingleBenchmarkAggregationChartFileMap = new HashMap<ProblemBenchmarkResult, List<File>>();
    CategoryAxis xAxis = new CategoryAxis("Solver Configurations");
    NumberAxis yAxis = new NumberAxis("Scores distribution of single benchmark runs");
    yAxis.setStandardTickUnits(NumberAxis.createIntegerTickUnits());
    yAxis.setAutoRangeIncludesZero(false);
    BoxAndWhiskerRenderer renderer =
        new BoxAndWhiskerRenderer() {
          @Override
          public int
              getRowCount() { // TODO: HACK for https://issues.jboss.org/browse/PLANNER-429 center
                              // plotted boxes to x axis labels
            return 1;
          }
        };
    renderer.setFillBox(true);
    renderer.setUseOutlinePaintForWhiskers(true);
    renderer.setMedianVisible(true);
    renderer.setMeanVisible(false);
    renderer.setItemMargin(0.0);

    for (ProblemBenchmarkResult problemBenchmarkResult :
        plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
      List<? extends BoxAndWhiskerCategoryDataset> datasetList =
          generateSubSingleBenchmarkScoreSummary(problemBenchmarkResult);
      List<File> chartFileList = new ArrayList<File>(datasetList.size());
      int scoreLevelIndex = 0;
      for (BoxAndWhiskerCategoryDataset dataset : datasetList) {
        CategoryPlot plot = new CategoryPlot(dataset, xAxis, yAxis, renderer);
        plot.setOrientation(PlotOrientation.VERTICAL);
        JFreeChart chart =
            new JFreeChart(
                problemBenchmarkResult
                    + " (level "
                    + scoreLevelIndex
                    + ") single benchmark runs score distribution",
                JFreeChart.DEFAULT_TITLE_FONT,
                plot,
                true);
        chartFileList.add(
            writeChartToImageFile(
                chart,
                "SubSingleSummary"
                    + problemBenchmarkResult.getAnchorId()
                    + "Level"
                    + scoreLevelIndex));
        scoreLevelIndex++;
      }
      subSingleBenchmarkAggregationChartFileMap.put(problemBenchmarkResult, chartFileList);
    }
  }
Exemplo n.º 2
0
 private void determineDefaultShownScoreLevelIndex() {
   defaultShownScoreLevelIndex = Integer.MAX_VALUE;
   for (ProblemBenchmarkResult problemBenchmarkResult :
       plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
     if (problemBenchmarkResult.hasAnySuccess()) {
       double[] winningScoreLevels =
           ScoreUtils.extractLevelDoubles(
               problemBenchmarkResult.getWinningSingleBenchmarkResult().getAverageScore());
       int[] differenceCount = new int[winningScoreLevels.length];
       for (int i = 0; i < differenceCount.length; i++) {
         differenceCount[i] = 0;
       }
       for (SingleBenchmarkResult singleBenchmarkResult :
           problemBenchmarkResult.getSingleBenchmarkResultList()) {
         if (singleBenchmarkResult.hasAllSuccess()) {
           double[] scoreLevels =
               ScoreUtils.extractLevelDoubles(singleBenchmarkResult.getAverageScore());
           for (int i = 0; i < scoreLevels.length; i++) {
             if (scoreLevels[i] != winningScoreLevels[i]) {
               differenceCount[i] = differenceCount[i] + 1;
             }
           }
         }
       }
       int firstInterestingLevel = differenceCount.length - 1;
       for (int i = 0; i < differenceCount.length; i++) {
         if (differenceCount[i] > 0) {
           firstInterestingLevel = i;
           break;
         }
       }
       if (defaultShownScoreLevelIndex > firstInterestingLevel) {
         defaultShownScoreLevelIndex = firstInterestingLevel;
       }
     }
   }
 }
Exemplo n.º 3
0
 private List<? extends BoxAndWhiskerCategoryDataset> generateSubSingleBenchmarkScoreSummary(
     ProblemBenchmarkResult problemBenchmarkResult) {
   List<DefaultBoxAndWhiskerCategoryDataset> datasetList =
       new ArrayList<DefaultBoxAndWhiskerCategoryDataset>(CHARTED_SCORE_LEVEL_SIZE);
   for (SingleBenchmarkResult singleBenchmarkResult :
       problemBenchmarkResult.getSingleBenchmarkResultList()) {
     List<List<Double>> valueListList = new ArrayList<List<Double>>(CHARTED_SCORE_LEVEL_SIZE);
     for (SubSingleBenchmarkResult subSingleBenchmarkResult :
         singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
       if (subSingleBenchmarkResult.hasAllSuccess() && subSingleBenchmarkResult.isInitialized()) {
         double[] levelValues =
             ScoreUtils.extractLevelDoubles(subSingleBenchmarkResult.getAverageScore());
         for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
           if (i >= valueListList.size()) {
             valueListList.add(new ArrayList<Double>(singleBenchmarkResult.getSuccessCount()));
           }
           valueListList.get(i).add(levelValues[i]);
         }
       }
     }
     for (int i = 0; i < valueListList.size() && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
       if (i >= datasetList.size()) {
         datasetList.add(new DefaultBoxAndWhiskerCategoryDataset());
       }
       SolverBenchmarkResult solverBenchmarkResult =
           singleBenchmarkResult.getSolverBenchmarkResult();
       datasetList
           .get(i)
           .add(
               valueListList.get(i),
               solverBenchmarkResult.getName(),
               solverBenchmarkResult
                   + " - "
                   + solverBenchmarkResult.getSubSingleCount()
                   + " run(s)");
     }
   }
   return datasetList;
 }
Exemplo n.º 4
0
 public void writeReport() {
   logger.info("Generating benchmark report...");
   summaryDirectory = new File(plannerBenchmarkResult.getBenchmarkReportDirectory(), "summary");
   summaryDirectory.mkdir();
   plannerBenchmarkResult.accumulateResults(this);
   fillWarningList();
   writeBestScoreSummaryCharts();
   writeBestScoreScalabilitySummaryChart();
   writeWinningScoreDifferenceSummaryChart();
   writeWorstScoreDifferencePercentageSummaryChart();
   writeAverageCalculateCountPerSecondSummaryChart();
   writeTimeSpentSummaryChart();
   writeTimeSpentScalabilitySummaryChart();
   writeBestScorePerTimeSpentSummaryChart();
   writeSubSingleBenchmarkScoreCharts();
   for (ProblemBenchmarkResult problemBenchmarkResult :
       plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
     for (SingleBenchmarkResult singleBenchmarkResult :
         problemBenchmarkResult.getSingleBenchmarkResultList()) {
       for (SubSingleBenchmarkResult subSingleBenchmarkResult :
           singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
         if (!subSingleBenchmarkResult.hasAllSuccess()) {
           continue;
         }
         for (SubSingleStatistic subSingleStatistic :
             subSingleBenchmarkResult.getEffectiveSubSingleStatisticMap().values()) {
           try {
             subSingleStatistic.unhibernatePointList();
           } catch (IllegalStateException e) {
             if (!plannerBenchmarkResult.getAggregation()) {
               throw new IllegalStateException(
                   "Failed to unhibernate point list of SubSingleStatistic ("
                       + subSingleStatistic
                       + ") of SubSingleBenchmark ("
                       + subSingleBenchmarkResult
                       + ").",
                   e);
             }
             logger.trace(
                 "This is expected, aggregator doesn't copy CSV files. Could not read CSV file "
                     + "({}) of sub single statistic ({}).",
                 subSingleStatistic.getCsvFile().getAbsolutePath(),
                 subSingleStatistic);
           }
         }
       }
     }
   }
   for (ProblemBenchmarkResult problemBenchmarkResult :
       plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
     if (problemBenchmarkResult.hasAnySuccess()) {
       for (ProblemStatistic problemStatistic : problemBenchmarkResult.getProblemStatisticList()) {
         problemStatistic.writeGraphFiles(this);
       }
       for (SingleBenchmarkResult singleBenchmarkResult :
           problemBenchmarkResult.getSingleBenchmarkResultList()) {
         if (singleBenchmarkResult.hasAllSuccess()) {
           for (PureSubSingleStatistic pureSubSingleStatistic :
               singleBenchmarkResult.getMedian().getPureSubSingleStatisticList()) {
             pureSubSingleStatistic.writeGraphFiles(this);
           }
         }
       }
     }
   }
   for (ProblemBenchmarkResult problemBenchmarkResult :
       plannerBenchmarkResult.getUnifiedProblemBenchmarkResultList()) {
     for (SingleBenchmarkResult singleBenchmarkResult :
         problemBenchmarkResult.getSingleBenchmarkResultList()) {
       for (SubSingleBenchmarkResult subSingleBenchmarkResult :
           singleBenchmarkResult.getSubSingleBenchmarkResultList()) {
         if (!subSingleBenchmarkResult.hasAllSuccess()) {
           continue;
         }
         for (SubSingleStatistic subSingleStatistic :
             subSingleBenchmarkResult.getEffectiveSubSingleStatisticMap().values()) {
           if (plannerBenchmarkResult.getAggregation()) {
             subSingleStatistic.setPointList(null);
           } else {
             subSingleStatistic.hibernatePointList();
           }
         }
       }
     }
   }
   determineDefaultShownScoreLevelIndex();
   writeHtmlOverviewFile();
 }