private void printResults(StabilityTrialResults results) {
System.out.println(
" Median = "
+ StabilityBenchmark.computePercent(results.breakingPoints, 0.5));
System.out.println(" Finished = " + results.getNumFinished());
System.out.println(" Bad Answer = " + results.getNumUncountable());
System.out.println(" Large Error = " + results.getNumLargeError());
System.out.println(" Detected = " + results.getNumGraceful());
System.out.println(" Runtime Exception = " + results.getNumUnexpectedException());
}
public void process() {
// MatrixLibrary libInfo = library.getLibrary();
for (StabilityTestBase op : operations) {
System.out.println(libraryName + " : Processing op: " + op.getTestName());
StabilityTrialResults results = evaluateOperation(op);
// if a fatal error occurred create some results so that this is marked
if (fatalError != null) {
results = new StabilityTrialResults();
logStream.println("Had fatal error: " + op.getTestName() + " error = " + fatalError);
results.fatalError = fatalError;
}
// add environmental information for debugging later on
results.durationMilli = tools.getDurationMilli();
results.libraryName = libraryName;
results.benchmarkName = op.getTestName();
results.memoryBytes = slaveMemoryMegaBytes;
// save the results to a file
saveResults(results, op.getFileName());
// print the results to the screen
if (fatalError == null) printResults(results);
}
if (directorySave != null) {
logStream.close();
}
}
public StabilityTrialResults evaluateOperation(StabilityTestBase e) {
fatalError = null;
e.setRandomSeed(config.randomSeed);
for (int attempts = 0; attempts < 5; attempts++) {
tools.setMemoryScale(attempts + 1);
EvaluatorSlave.Results results = spawnChild ? tools.runTest(e) : tools.runTestNoSpawn(e);
slaveMemoryMegaBytes = tools.getAllocatedMemory();
if (results == null) {
logStream.println("*** WTF runTest returned null = " + e.getTestName());
fatalError = FatalError.RETURNED_NULL;
} else if (results.failed == EvaluatorSlave.FailReason.USER_REQUESTED) {
logStream.println(
" Slave was killed by the user/OS. Stopping the benchmark. op = "
+ e.getTestName());
logStream.println(" error message: " + results.detailedError);
System.out.println(" Slave was killed by the user/OS. Stopping the benchmark.");
System.out.println(" error message: " + results.detailedError);
System.exit(0);
} else if (results.failed == EvaluatorSlave.FailReason.OUT_OF_MEMORY) {
System.out.println(" Not enough memory given to slave. Attempt " + attempts);
logStream.println(
"Not enough memory for op. Attempt num "
+ attempts
+ " op = "
+ e.getTestName()
+ " memory "
+ tools.getAllocatedMemory());
// have it run again, which will up the memory
continue;
} else {
if (results.failed != null) {
fatalError = FatalError.MISC;
if (results.failed == EvaluatorSlave.FailReason.TOO_SLOW) {
logStream.println(" Slave: Case too slow = " + e.getTestName());
} else if (results.failed == EvaluatorSlave.FailReason.FROZEN) {
logStream.println(" Slave: Frozen = " + e.getTestName());
fatalError = FatalError.FROZE;
} else {
logStream.println(
" Slave: Case failed = "
+ results.failed
+ " op = "
+ e.getTestName()
+ " memory "
+ tools.getAllocatedMemory());
if (results.detailedError != null) {
logStream.println(results.detailedError);
}
}
}
}
// see if something very bad happened
if (fatalError != null) return null;
// collect all the results and return them
StabilityTrialResults all = new StabilityTrialResults();
for (TestResults t : results.getResults()) {
StabilityTrialResults r = (StabilityTrialResults) t;
all.addResults(r);
}
return all;
}
fatalError = FatalError.OUT_OF_MEMORY;
return null;
}
