@Override
public void interpretArguments(
final JSAP jsap, final JSAPResult result, final DAVOptions options) {
timeService = new TimeLoggingService("execute-splits");
timeService.start();
super.interpretArguments(jsap, result, options);
optionalModelIds = parseOptionalModelIdProperties(configurationProperties);
evaluateStatistics = result.getBoolean("evaluate-statistics");
if (!evaluateStatistics) {
System.out.println(
"Will not evaluate statistics as run proceed. Run restat on results directory to obtain statistics.");
} else {
System.out.println("Will evaluate statistics as run proceed.");
}
try {
final String filename = result.getString("splits");
splitPlanFilename = filename;
splitPlan.load(filename);
} catch (Exception e) {
LOGGER.error("An error occurred reading splits file. " + result.getString("splits"), e);
}
// collect keys for parameters that are not required (have default values)
paramKeysNotRequired = new ObjectArrayList<String>();
if (!result.userSpecified("seed")) {
paramKeysNotRequired.add("--seed");
paramKeysNotRequired.add(Integer.toString(options.randomSeed));
}
modelId = ShortHash.shortHash(getOriginalArgs());
options.modelId = modelId;
timeService.setModelId(options.modelId);
final Map<String, String> additionalConditionsMap = new HashMap<String, String>();
additionalConditionsMap.put("model-id", modelId);
for (final OptionalModelId optionalModelId : optionalModelIds) {
final String[] originalArgs1 = expandShortArgs(getOriginalArgs(), jsap);
final String[] filteredArgs = filterArgs(originalArgs1, optionalModelId);
final String optionalModelIdValue = ShortHash.shortHash(filteredArgs);
additionalConditionsMap.put(optionalModelId.columnIdentifier, optionalModelIdValue);
}
final String modelConditionsFilename = "model-conditions.txt";
final Set<String> skipJsapConditions = new HashSet<String>();
skipJsapConditions.add("model-id");
skipJsapConditions.add("mode");
try {
writeConditions(
modelConditionsFilename, jsap, result, additionalConditionsMap, skipJsapConditions);
} catch (IOException e) {
LOGGER.error("Error writing " + modelConditionsFilename + " file", e);
}
}
/**
* Parse properties to extract optional model id definitions. The format is as follow:
*
* <p>
*
* <PRE>
* define.model-id.column-id=modelid-noScaler
* define.model-id.modelid-noScaler.exclude=a,b
* define.model-id.modelid-noScaler.exclude.a.argument=scaler-name
* define.model-id.modelid-noScaler.exclude.a.skip=1
* define.model-id.modelid-noScaler.exclude.b.argument=normalizer-name
* define.model-id.modelid-noScaler.exclude.b.skip=1
* </PRE>
*
* These properties would define one new model-id called, to be written in a column called
* modelid-noScaler, which excludes two arguments and one value each from the hashcode modelId
* calculation.
*/
public static OptionalModelId[] parseOptionalModelIdProperties(
final Properties configurationProperties) {
final ObjectList<OptionalModelId> result = new ObjectArrayList<OptionalModelId>();
if (configurationProperties != null) {
// inspect properties to figure out which optional model ids to create:
final ObjectSet<String> optionalModelIdColumnNames = new ObjectArraySet<String>();
for (final String propertyName : configurationProperties.stringPropertyNames()) {
if (propertyName.startsWith("define.model-id.column-id")) {
final String columnIdNames = configurationProperties.getProperty(propertyName);
final String[] names = columnIdNames.split(",");
for (final String name : names) {
optionalModelIdColumnNames.add(name);
}
}
}
for (final String optionalColumnId : optionalModelIdColumnNames) {
final String defineModelIdExcludePropertyName =
"define.model-id." + optionalColumnId + ".exclude";
final String argumentKeys =
configurationProperties.getProperty(defineModelIdExcludePropertyName);
final String[] keys;
if (argumentKeys == null) {
System.err.println(
"Error parsing properties. Cannot find key=" + defineModelIdExcludePropertyName);
keys = ArrayUtils.EMPTY_STRING_ARRAY;
} else {
keys = argumentKeys.split(",");
}
final OptionalModelId newOne = new OptionalModelId(optionalColumnId);
for (String key : keys) {
key = key.trim();
final String excludeArgumentName =
configurationProperties.getProperty(
defineModelIdExcludePropertyName + "." + key + ".argument");
final String excludeArgumentSkip =
configurationProperties.getProperty(
defineModelIdExcludePropertyName + "." + key + ".skip");
newOne.addExcludeArgument(excludeArgumentName, Integer.parseInt(excludeArgumentSkip));
}
result.add(newOne);
LOGGER.debug("Defined modelId: " + newOne);
}
}
return result.toArray(new OptionalModelId[result.size()]);
}
private String[] filterArgs(final String[] originalArgs, final OptionalModelId optionalModelId) {
final ObjectList<String> filteredArgs = new ObjectArrayList<String>();
for (int i = 0; i < originalArgs.length; i++) {
final String argumentName = originalArgs[i].replaceAll("--", "");
if (optionalModelId.columnIdentifier.equalsIgnoreCase(
argumentName)) { // skip optional modelIds arguments as well.
final int skipNumber = 1;
LOGGER.info(
"For optional modelId: "
+ optionalModelId.columnIdentifier
+ " Filtering out argument "
+ argumentName
+ " total args skipped: "
+ skipNumber);
i += skipNumber; // skip argument name and 'skip' number of arguments.
}
if (optionalModelId.excludeArgumentNames.contains(argumentName)) {
final int skipNumber = optionalModelId.skipValue(argumentName);
LOGGER.info(
"For optional modelId: "
+ optionalModelId.columnIdentifier
+ " Filtering out argument "
+ argumentName
+ " total args skipped: "
+ skipNumber);
i += skipNumber; // skip argument name and 'skip' number of arguments.
} else {
LOGGER.debug("Adding argument to hashCode: " + originalArgs[i]);
filteredArgs.add(originalArgs[i]);
}
if (i >= originalArgs.length) {
break;
}
}
// Hashcode will depend on argument order, so we sort them after filtering:
Collections.sort(filteredArgs);
return filteredArgs.toArray(new String[filteredArgs.size()]);
}
private String[] adjustArguments(
final String[] args, final int splitIndex, final ObjectList<String> explicitOptions) {
final ObjectList<String> filteredArgs = new ObjectArrayList<String>();
int ignoreCount = 0;
for (final String arg : args) {
if (arg.equals("--sequence-file")) {
ignoreCount = 1;
filteredArgs.add("--sequence-file");
}
if (arg.equals("--mode") || arg.equals("-m")) {
ignoreCount = 2;
filteredArgs.add("--mode");
filteredArgs.add("sequence");
}
if (arg.equals("--splits")) {
ignoreCount = 2;
}
if (arg.equals("--evaluate-statistics")) {
ignoreCount = 2;
}
if (ignoreCount > 0) {
ignoreCount--;
} else {
filteredArgs.add(arg);
}
}
filteredArgs.add("--process-split-id");
filteredArgs.add(Integer.toString(splitIndex));
filteredArgs.add("--split-id");
filteredArgs.add(Integer.toString(splitIndex));
filteredArgs.add("--model-id");
filteredArgs.add(modelId);
filteredArgs.add("--split-plan");
filteredArgs.add(splitPlanFilename);
filteredArgs.add("--other-options");
final StringBuilder argOtherOptions = new StringBuilder();
for (final String option : explicitOptions) {
argOtherOptions.append(option);
argOtherOptions.append(' ');
}
filteredArgs.add(argOtherOptions.toString());
return filteredArgs.toArray(new String[filteredArgs.size()]);
}
public void addRemovalListener(RemovalListener removalListener) {
removalListeners.add(removalListener);
}