protected static boolean hasRepresentation( final String operation, Map<Report, List<Aggregation>> reportAggregationMap, final Class<?> representationClass, final Object... representationArgs) { return Projections.any( Projections.notNull(reportAggregationMap.values()), new Projections.Condition<List<Aggregation>>() { @Override public boolean accept(List<Aggregation> aggregations) { return Projections.any( Projections.notNull(aggregations), new Projections.Condition<Aggregation>() { @Override public boolean accept(Aggregation aggregation) { OperationStats operationStats = aggregation.totalStats.getOperationsStats().get(operation); return operationStats != null && operationStats.getRepresentation(representationClass, representationArgs) != null; } }); } }); }
public int getMaxThreads(List<Aggregation> aggregations, final int slaveIndex) { Integer maxThreads = Projections.max( Projections.project( aggregations, new Projections.Func<Aggregation, Integer>() { @Override public Integer project(Aggregation aggregation) { List<Statistics> statistics = aggregation.iteration.getStatistics(slaveIndex); return statistics == null ? 0 : statistics.size(); } })); return maxThreads != null ? maxThreads : 0; }
@Override public StageResult processAckOnMaster(List<DistStageAck> acks) { StageResult result = super.processAckOnMaster(acks); if (result.isError()) return result; log.info("--------------------"); if (ramPercentage > 0) { if (stringData) { log.info( "Filled cache with String objects totaling " + Math.round(ramPercentage * 100) + "% of the Java heap"); } else { log.info( "Filled cache with byte arrays totaling " + Math.round(ramPercentage * 100) + "% of the Java heap"); } } if (ramPercentage < 0 && targetMemoryUse > 0) { if (stringData) { log.info("Filled cache with String objects totaling " + Utils.kbString(targetMemoryUse)); } else { log.info("Filled cache with byte arrays totaling " + Utils.kbString(targetMemoryUse)); } } if (valueCount > 0) { if (stringData) { log.info( "Filled cache with " + Utils.kbString((valueSize * 2) * valueCount) + " of " + valueSize + " character String objects"); } else { log.info( "Filled cache with " + Utils.kbString(valueSize * valueCount) + " of " + Utils.kbString(valueSize) + " byte arrays"); } } Report report = masterState.getReport(); Report.Test test = report.createTest("Random_Data_Stage", null, true); int testIteration = test.getIterations().size(); Map<Integer, Report.SlaveResult> nodeKeyCountsResult = new HashMap<Integer, Report.SlaveResult>(); Map<Integer, Report.SlaveResult> nodeTargetMemoryUseResult = new HashMap<Integer, Report.SlaveResult>(); Map<Integer, Report.SlaveResult> nodeCountOfWordsInDataResult = new HashMap<Integer, Report.SlaveResult>(); Map<Integer, Report.SlaveResult> nodeBytesWritten = new HashMap<Integer, Report.SlaveResult>(); long totalValues = 0; long totalBytes = 0; long totalNodeWordCount = 0; Map<String, Integer> clusterWordCount = new TreeMap<String, Integer>(); for (DataInsertAck ack : Projections.instancesOf(acks, DataInsertAck.class)) { if (ack.wordCount != null) { for (Map.Entry<String, Integer> entry : ack.wordCount.entrySet()) { if (clusterWordCount.containsKey(entry.getKey())) { clusterWordCount.put( entry.getKey(), clusterWordCount.get(entry.getKey()) + entry.getValue()); } else { clusterWordCount.put(entry.getKey(), entry.getValue()); } } } nodeKeyCountsResult.put( ack.getSlaveIndex(), new Report.SlaveResult(Long.toString(ack.nodeKeyCount), false)); nodeBytesWritten.put( ack.getSlaveIndex(), new Report.SlaveResult(Long.toString(ack.bytesWritten), false)); test.addStatistics( testIteration, ack.getSlaveIndex(), Collections.singletonList(ack.nodePutStats)); totalValues += ack.nodePutCount; totalBytes += ack.bytesWritten; String logInfo = "Slave " + ack.getSlaveIndex() + " wrote " + ack.nodePutCount + " values to the cache with a total size of " + Utils.kbString(ack.bytesWritten); if (ramPercentage > 0) { logInfo += "; targetMemoryUse = " + Utils.kbString(ack.targetMemoryUse); nodeTargetMemoryUseResult.put( ack.getSlaveIndex(), new Report.SlaveResult(Long.toString(ack.targetMemoryUse), false)); } if (stringData) { logInfo += "; countOfWordsInData = " + ack.countOfWordsInData; nodeCountOfWordsInDataResult.put( ack.getSlaveIndex(), new Report.SlaveResult(Long.toString(ack.countOfWordsInData), false)); } log.info(logInfo); } log.info( "The cache contains " + totalValues + " values with a total size of " + Utils.kbString(totalBytes)); if (limitWordCount) { int totalWordCount = maxWordCount; if (!shareWords) { totalWordCount = maxWordCount * slaveState.getClusterSize(); } log.info( "Up to " + totalWordCount + " words were generated with a maximum length of " + maxWordLength + " characters"); } if (!clusterWordCount.isEmpty()) { log.info(clusterWordCount.size() + "words were actually generated"); log.info("--------------------"); log.info("Cluster wide word count:"); for (String key : clusterWordCount.keySet()) { log.info("word: " + key + "; count: " + clusterWordCount.get(key)); } // TODO Will this take too much memory? // masterState.put(RANDOMDATA_CLUSTER_WORDCOUNT_KEY, clusterWordCount); } log.info("--------------------"); masterState.put(RANDOMDATA_TOTALBYTES_KEY, totalBytes); test.addResult( testIteration, new Report.TestResult( "Kilobytes written per node", nodeBytesWritten, Utils.kbString(totalBytes), false)); test.addResult( testIteration, new Report.TestResult("Key count per node", nodeKeyCountsResult, "", false)); if (!nodeTargetMemoryUseResult.isEmpty()) { test.addResult( testIteration, new Report.TestResult( "Target memory use per node", nodeTargetMemoryUseResult, Utils.kbString(totalBytes), false)); } if (!nodeCountOfWordsInDataResult.isEmpty()) { test.addResult( testIteration, new Report.TestResult( "Count of words in data per node", nodeCountOfWordsInDataResult, Long.toString(totalNodeWordCount), false)); } return StageResult.SUCCESS; }
protected boolean addToChart( ComparisonChart chart, String subCategory, String operation, ChartType chartType, Map<Report, List<Aggregation>> reportAggregationMap) { Map<String, List<Report>> byConfiguration = Projections.groupBy( reportAggregationMap.keySet(), new Projections.Func<Report, String>() { @Override public String project(Report report) { return report.getConfiguration().name; } }); for (Map.Entry<Report, List<Aggregation>> entry : reportAggregationMap.entrySet()) { for (Aggregation aggregation : entry.getValue()) { OperationStats operationStats = aggregation.totalStats.getOperationsStats().get(operation); if (operationStats == null) return false; String categoryName = entry.getKey().getConfiguration().name; if (subCategory != null) { categoryName = String.format("%s, %s", categoryName, subCategory); } // if there are multiple reports for the same configuration (multiple clusters), use cluster // size in category if (byConfiguration.get(entry.getKey().getConfiguration().name).size() > 1) { categoryName = String.format("%s, size %d", categoryName, entry.getKey().getCluster().getSize()); } double subCategoryNumeric; String subCategoryValue; if (maxIterations > 1) { subCategoryNumeric = aggregation.iteration.id; subCategoryValue = aggregation.iteration.getValue() != null ? aggregation.iteration.getValue() : String.valueOf(aggregation.iteration.id); } else { subCategoryNumeric = entry.getKey().getCluster().getSize(); subCategoryValue = String.format("Size %.0f", subCategoryNumeric); } switch (chartType) { case MEAN_AND_DEV: { MeanAndDev meanAndDev = operationStats.getRepresentation(MeanAndDev.class); if (meanAndDev == null) return false; chart.addValue( toMillis(meanAndDev.mean), toMillis(meanAndDev.dev), categoryName, subCategoryNumeric, subCategoryValue); break; } case OPERATION_THROUGHPUT_GROSS: { OperationThroughput throughput = operationStats.getRepresentation( OperationThroughput.class, TimeUnit.MILLISECONDS.toNanos( aggregation.totalStats.getEnd() - aggregation.totalStats.getBegin())); if (throughput == null) return false; chart.addValue( throughput.gross, 0, categoryName, subCategoryNumeric, subCategoryValue); break; } case OPERATION_THROUGHPUT_NET: { OperationThroughput throughput = operationStats.getRepresentation( OperationThroughput.class, TimeUnit.MILLISECONDS.toNanos( aggregation.totalStats.getEnd() - aggregation.totalStats.getBegin())); if (throughput == null) return false; chart.addValue(throughput.net, 0, categoryName, subCategoryNumeric, subCategoryValue); break; } case DATA_THROUGHPUT: { DataThroughput dataThroughput = operationStats.getRepresentation( DataThroughput.class, aggregation.totalThreads, aggregation.totalStats.getEnd() - aggregation.totalStats.getBegin()); if (dataThroughput == null) return false; chart.addValue( dataThroughput.meanThroughput / (1024.0 * 1024.0), dataThroughput.deviation / (1024.0 * 1024.0), categoryName, subCategoryNumeric, subCategoryValue); break; } } } } return true; }