예제 #1
0
 protected static boolean hasRepresentation(
     final String operation,
     Map<Report, List<Aggregation>> reportAggregationMap,
     final Class<?> representationClass,
     final Object... representationArgs) {
   return Projections.any(
       Projections.notNull(reportAggregationMap.values()),
       new Projections.Condition<List<Aggregation>>() {
         @Override
         public boolean accept(List<Aggregation> aggregations) {
           return Projections.any(
               Projections.notNull(aggregations),
               new Projections.Condition<Aggregation>() {
                 @Override
                 public boolean accept(Aggregation aggregation) {
                   OperationStats operationStats =
                       aggregation.totalStats.getOperationsStats().get(operation);
                   return operationStats != null
                       && operationStats.getRepresentation(representationClass, representationArgs)
                           != null;
                 }
               });
         }
       });
 }
예제 #2
0
 public int getMaxThreads(List<Aggregation> aggregations, final int slaveIndex) {
   Integer maxThreads =
       Projections.max(
           Projections.project(
               aggregations,
               new Projections.Func<Aggregation, Integer>() {
                 @Override
                 public Integer project(Aggregation aggregation) {
                   List<Statistics> statistics = aggregation.iteration.getStatistics(slaveIndex);
                   return statistics == null ? 0 : statistics.size();
                 }
               }));
   return maxThreads != null ? maxThreads : 0;
 }
예제 #3
0
  @Override
  public StageResult processAckOnMaster(List<DistStageAck> acks) {
    StageResult result = super.processAckOnMaster(acks);
    if (result.isError()) return result;

    log.info("--------------------");
    if (ramPercentage > 0) {
      if (stringData) {
        log.info(
            "Filled cache with String objects totaling "
                + Math.round(ramPercentage * 100)
                + "% of the Java heap");
      } else {
        log.info(
            "Filled cache with byte arrays totaling "
                + Math.round(ramPercentage * 100)
                + "% of the Java heap");
      }
    }
    if (ramPercentage < 0 && targetMemoryUse > 0) {
      if (stringData) {
        log.info("Filled cache with String objects totaling " + Utils.kbString(targetMemoryUse));
      } else {
        log.info("Filled cache with byte arrays totaling " + Utils.kbString(targetMemoryUse));
      }
    }
    if (valueCount > 0) {
      if (stringData) {
        log.info(
            "Filled cache with "
                + Utils.kbString((valueSize * 2) * valueCount)
                + " of "
                + valueSize
                + " character String objects");
      } else {
        log.info(
            "Filled cache with "
                + Utils.kbString(valueSize * valueCount)
                + " of "
                + Utils.kbString(valueSize)
                + " byte arrays");
      }
    }

    Report report = masterState.getReport();
    Report.Test test = report.createTest("Random_Data_Stage", null, true);
    int testIteration = test.getIterations().size();

    Map<Integer, Report.SlaveResult> nodeKeyCountsResult =
        new HashMap<Integer, Report.SlaveResult>();
    Map<Integer, Report.SlaveResult> nodeTargetMemoryUseResult =
        new HashMap<Integer, Report.SlaveResult>();
    Map<Integer, Report.SlaveResult> nodeCountOfWordsInDataResult =
        new HashMap<Integer, Report.SlaveResult>();
    Map<Integer, Report.SlaveResult> nodeBytesWritten = new HashMap<Integer, Report.SlaveResult>();

    long totalValues = 0;
    long totalBytes = 0;
    long totalNodeWordCount = 0;
    Map<String, Integer> clusterWordCount = new TreeMap<String, Integer>();
    for (DataInsertAck ack : Projections.instancesOf(acks, DataInsertAck.class)) {
      if (ack.wordCount != null) {
        for (Map.Entry<String, Integer> entry : ack.wordCount.entrySet()) {
          if (clusterWordCount.containsKey(entry.getKey())) {
            clusterWordCount.put(
                entry.getKey(), clusterWordCount.get(entry.getKey()) + entry.getValue());
          } else {
            clusterWordCount.put(entry.getKey(), entry.getValue());
          }
        }
      }

      nodeKeyCountsResult.put(
          ack.getSlaveIndex(), new Report.SlaveResult(Long.toString(ack.nodeKeyCount), false));
      nodeBytesWritten.put(
          ack.getSlaveIndex(), new Report.SlaveResult(Long.toString(ack.bytesWritten), false));
      test.addStatistics(
          testIteration, ack.getSlaveIndex(), Collections.singletonList(ack.nodePutStats));

      totalValues += ack.nodePutCount;
      totalBytes += ack.bytesWritten;
      String logInfo =
          "Slave "
              + ack.getSlaveIndex()
              + " wrote "
              + ack.nodePutCount
              + " values to the cache with a total size of "
              + Utils.kbString(ack.bytesWritten);
      if (ramPercentage > 0) {
        logInfo += "; targetMemoryUse = " + Utils.kbString(ack.targetMemoryUse);
        nodeTargetMemoryUseResult.put(
            ack.getSlaveIndex(), new Report.SlaveResult(Long.toString(ack.targetMemoryUse), false));
      }
      if (stringData) {
        logInfo += "; countOfWordsInData = " + ack.countOfWordsInData;
        nodeCountOfWordsInDataResult.put(
            ack.getSlaveIndex(),
            new Report.SlaveResult(Long.toString(ack.countOfWordsInData), false));
      }
      log.info(logInfo);
    }
    log.info(
        "The cache contains "
            + totalValues
            + " values with a total size of "
            + Utils.kbString(totalBytes));
    if (limitWordCount) {
      int totalWordCount = maxWordCount;
      if (!shareWords) {
        totalWordCount = maxWordCount * slaveState.getClusterSize();
      }
      log.info(
          "Up to "
              + totalWordCount
              + " words were generated with a maximum length of "
              + maxWordLength
              + " characters");
    }
    if (!clusterWordCount.isEmpty()) {
      log.info(clusterWordCount.size() + "words were actually generated");
      log.info("--------------------");
      log.info("Cluster wide word count:");
      for (String key : clusterWordCount.keySet()) {
        log.info("word: " + key + "; count: " + clusterWordCount.get(key));
      }
      // TODO Will this take too much memory?
      //         masterState.put(RANDOMDATA_CLUSTER_WORDCOUNT_KEY, clusterWordCount);
    }
    log.info("--------------------");

    masterState.put(RANDOMDATA_TOTALBYTES_KEY, totalBytes);
    test.addResult(
        testIteration,
        new Report.TestResult(
            "Kilobytes written per node", nodeBytesWritten, Utils.kbString(totalBytes), false));

    test.addResult(
        testIteration, new Report.TestResult("Key count per node", nodeKeyCountsResult, "", false));
    if (!nodeTargetMemoryUseResult.isEmpty()) {
      test.addResult(
          testIteration,
          new Report.TestResult(
              "Target memory use per node",
              nodeTargetMemoryUseResult,
              Utils.kbString(totalBytes),
              false));
    }
    if (!nodeCountOfWordsInDataResult.isEmpty()) {
      test.addResult(
          testIteration,
          new Report.TestResult(
              "Count of words in data per node",
              nodeCountOfWordsInDataResult,
              Long.toString(totalNodeWordCount),
              false));
    }

    return StageResult.SUCCESS;
  }
예제 #4
0
  protected boolean addToChart(
      ComparisonChart chart,
      String subCategory,
      String operation,
      ChartType chartType,
      Map<Report, List<Aggregation>> reportAggregationMap) {
    Map<String, List<Report>> byConfiguration =
        Projections.groupBy(
            reportAggregationMap.keySet(),
            new Projections.Func<Report, String>() {
              @Override
              public String project(Report report) {
                return report.getConfiguration().name;
              }
            });
    for (Map.Entry<Report, List<Aggregation>> entry : reportAggregationMap.entrySet()) {
      for (Aggregation aggregation : entry.getValue()) {
        OperationStats operationStats = aggregation.totalStats.getOperationsStats().get(operation);
        if (operationStats == null) return false;

        String categoryName = entry.getKey().getConfiguration().name;
        if (subCategory != null) {
          categoryName = String.format("%s, %s", categoryName, subCategory);
        }
        // if there are multiple reports for the same configuration (multiple clusters), use cluster
        // size in category
        if (byConfiguration.get(entry.getKey().getConfiguration().name).size() > 1) {
          categoryName =
              String.format("%s, size %d", categoryName, entry.getKey().getCluster().getSize());
        }

        double subCategoryNumeric;
        String subCategoryValue;
        if (maxIterations > 1) {
          subCategoryNumeric = aggregation.iteration.id;
          subCategoryValue =
              aggregation.iteration.getValue() != null
                  ? aggregation.iteration.getValue()
                  : String.valueOf(aggregation.iteration.id);
        } else {
          subCategoryNumeric = entry.getKey().getCluster().getSize();
          subCategoryValue = String.format("Size %.0f", subCategoryNumeric);
        }
        switch (chartType) {
          case MEAN_AND_DEV:
            {
              MeanAndDev meanAndDev = operationStats.getRepresentation(MeanAndDev.class);
              if (meanAndDev == null) return false;
              chart.addValue(
                  toMillis(meanAndDev.mean),
                  toMillis(meanAndDev.dev),
                  categoryName,
                  subCategoryNumeric,
                  subCategoryValue);
              break;
            }
          case OPERATION_THROUGHPUT_GROSS:
            {
              OperationThroughput throughput =
                  operationStats.getRepresentation(
                      OperationThroughput.class,
                      TimeUnit.MILLISECONDS.toNanos(
                          aggregation.totalStats.getEnd() - aggregation.totalStats.getBegin()));
              if (throughput == null) return false;
              chart.addValue(
                  throughput.gross, 0, categoryName, subCategoryNumeric, subCategoryValue);
              break;
            }
          case OPERATION_THROUGHPUT_NET:
            {
              OperationThroughput throughput =
                  operationStats.getRepresentation(
                      OperationThroughput.class,
                      TimeUnit.MILLISECONDS.toNanos(
                          aggregation.totalStats.getEnd() - aggregation.totalStats.getBegin()));
              if (throughput == null) return false;
              chart.addValue(throughput.net, 0, categoryName, subCategoryNumeric, subCategoryValue);
              break;
            }
          case DATA_THROUGHPUT:
            {
              DataThroughput dataThroughput =
                  operationStats.getRepresentation(
                      DataThroughput.class,
                      aggregation.totalThreads,
                      aggregation.totalStats.getEnd() - aggregation.totalStats.getBegin());
              if (dataThroughput == null) return false;
              chart.addValue(
                  dataThroughput.meanThroughput / (1024.0 * 1024.0),
                  dataThroughput.deviation / (1024.0 * 1024.0),
                  categoryName,
                  subCategoryNumeric,
                  subCategoryValue);
              break;
            }
        }
      }
    }
    return true;
  }