@Override
public void interpretArguments(
final JSAP jsap, final JSAPResult result, final DAVOptions options) {
timeService = new TimeLoggingService("execute-splits");
timeService.start();
super.interpretArguments(jsap, result, options);
optionalModelIds = parseOptionalModelIdProperties(configurationProperties);
evaluateStatistics = result.getBoolean("evaluate-statistics");
if (!evaluateStatistics) {
System.out.println(
"Will not evaluate statistics as run proceed. Run restat on results directory to obtain statistics.");
} else {
System.out.println("Will evaluate statistics as run proceed.");
}
try {
final String filename = result.getString("splits");
splitPlanFilename = filename;
splitPlan.load(filename);
} catch (Exception e) {
LOGGER.error("An error occurred reading splits file. " + result.getString("splits"), e);
}
// collect keys for parameters that are not required (have default values)
paramKeysNotRequired = new ObjectArrayList<String>();
if (!result.userSpecified("seed")) {
paramKeysNotRequired.add("--seed");
paramKeysNotRequired.add(Integer.toString(options.randomSeed));
}
modelId = ShortHash.shortHash(getOriginalArgs());
options.modelId = modelId;
timeService.setModelId(options.modelId);
final Map<String, String> additionalConditionsMap = new HashMap<String, String>();
additionalConditionsMap.put("model-id", modelId);
for (final OptionalModelId optionalModelId : optionalModelIds) {
final String[] originalArgs1 = expandShortArgs(getOriginalArgs(), jsap);
final String[] filteredArgs = filterArgs(originalArgs1, optionalModelId);
final String optionalModelIdValue = ShortHash.shortHash(filteredArgs);
additionalConditionsMap.put(optionalModelId.columnIdentifier, optionalModelIdValue);
}
final String modelConditionsFilename = "model-conditions.txt";
final Set<String> skipJsapConditions = new HashSet<String>();
skipJsapConditions.add("model-id");
skipJsapConditions.add("mode");
try {
writeConditions(
modelConditionsFilename, jsap, result, additionalConditionsMap, skipJsapConditions);
} catch (IOException e) {
LOGGER.error("Error writing " + modelConditionsFilename + " file", e);
}
}
@Override
public void process(final DAVOptions options) {
final String[] args = getOriginalArgs();
final UseModality<DAVOptions> executed;
final int maxSplitIndex = splitPlan.getMaxSplitIndex();
final ProgressLogger logger = new ProgressLogger(LOGGER);
logger.expectedUpdates = maxSplitIndex;
logger.itemsName = "splits";
logger.priority = Level.INFO;
logger.start("Parallel split processing");
final SplitParallelRegion region = new SplitParallelRegion(maxSplitIndex, args, logger);
try {
getParallelTeam().execute(region);
} catch (Exception e) {
LOGGER.error("An exception occurred.", e);
}
logger.stop();
/** Time the duration of the sequence: */
timeService.setModelId(modelId);
timeService.stop();
executed = region.getExecuted();
if (executed != null && executed instanceof SequenceMode) {
// if we executed SequenceMode
final SequenceMode sequenceMode = (SequenceMode) executed;
if (evaluateStatistics) {
final String label = sequenceMode.getValue("label");
final String statsFilename = sequenceMode.getValue("predictions-filename");
if (statsFilename != null && label != null) {
// and the sequence defined the variables "predictions-filename" and "label"
try {
final List<String> statsModeArgs =
new ObjectArrayList<String>(
new String[] {
"--mode",
"stats",
"--predictions",
statsFilename,
"--submission-file",
labelPrefix(label) + "-maqcii-submission.txt",
"--label",
label,
"--model-id",
modelId,
"--dataset-name",
options.datasetName,
"--other-measures",
"prec,rec,F-1,MCC,binary-auc"
});
if (options.adjustSignalToFloorValue) {
statsModeArgs.add("--floor");
statsModeArgs.add(Double.toString(options.signalFloorValue));
}
// extract survival options if any
// TODO: clean this up - we should not be checking for "%survival%"
final String survivalFileName = sequenceMode.getValue("survival");
if (StringUtils.isNotBlank(survivalFileName)
&& !"%survival%".equals(survivalFileName)) {
statsModeArgs.add("--survival");
statsModeArgs.add(survivalFileName);
}
LOGGER.debug("Estimating statistics: " + statsModeArgs);
// we create a new DAVMode here since we want to use the old StatsMode code
// which is no longer exposed by DiscoverAndValidate (BDVal main method)
final DAVMode statsMode = new DAVMode();
statsMode.registerMode("stats", StatsMode.class);
final DAVOptions statsModeOptions = new DAVOptions();
statsMode.process(
statsModeArgs.toArray(new String[statsModeArgs.size()]), statsModeOptions);
} catch (Exception e) {
LOGGER.error("Error executing --mode stats for all splits", e);
}
}
}
}
}
