private void SaveCrossValidationAssignments() throws Exception {
String outerFoldsFilePath =
Settings.GetOutputSettingsDir(true) + "Validation_Assignments_for_Outer_Folds.txt";
FileUtilities.WriteTextToFile(
outerFoldsFilePath,
"# The outer cross-validation fold to which each data instance was assigned.\n");
FileUtilities.AppendTextToFile(
outerFoldsFilePath, Singletons.InstanceVault.GetCrossValidationAssignments().toString());
if (Singletons.InstanceVault.GetCrossValidationAssignments().NumFolds
!= Singletons.InstanceVault.TransformedDependentVariableInstances.Size())
for (int outerFold :
Singletons.InstanceVault.GetCrossValidationAssignments().GetAllFoldNumbers()) {
String innerFoldFilePath =
Settings.GetOutputSettingsDir(true)
+ "Validation_Assignments_for_Inner_Folds_in_OuterFold_"
+ outerFold
+ ".txt";
FileUtilities.WriteTextToFile(
innerFoldFilePath,
"# The inner cross-validation fold (within outer fold "
+ outerFold
+ ") to which data instances were assigned.\n");
FileUtilities.AppendTextToFile(
innerFoldFilePath,
Singletons.InstanceVault.GetCrossValidationAssignments()
.GetInnerAssignments(outerFold)
.toString());
}
}
private void SaveExcludedTrainingIDInfo() throws Exception {
ArrayList<String> excludedTrainingIDs =
Singletons.InstanceVault.GetCrossValidationAssignments().GetAllExcludedTrainIDs();
if (excludedTrainingIDs.size() > 0) {
String filePath = Settings.GetOutputSettingsDir(true) + "Excluded_Training_IDs.txt";
String output = ListUtilities.Join(excludedTrainingIDs, "\n");
FileUtilities.WriteLineToFile(
filePath,
"# IDs of data instances that were excluded randomly from training sets, according to the specified experiment settings. If this is a cross-validation experiment, the instances were removed from different folds.");
FileUtilities.AppendLineToFile(filePath, output);
}
}
/**
* Depending on the settings for the experiment and the current iteration, this method returns a
* path to where the settings files should be saved.
*
* @param appendIterationIfMoreThanOne Whether to append the iteration number if more than one
* iteration is being executed
* @return Path to where the settings files should be saved
* @throws Exception
*/
public static String GetOutputSettingsDir(boolean appendIterationIfMoreThanOne) throws Exception {
String dirPath = Settings.OUTPUT_DIR + "Settings/";
if (Singletons.Config.GetNumIterations() > 1 && appendIterationIfMoreThanOne)
dirPath += "Iteration" + Singletons.Iteration + "/";
return FileUtilities.CreateDirectoryIfNotExists(dirPath);
}
/**
* The purpose of this method is to parse the information that is listed in the learners
* configuration file.
*
* @throws Exception
*/
public static void ParseLearners() throws Exception {
for (String line : FileUtilities.ReadLinesFromFile(LEARNER_TEMPLATES_FILE)) {
if (line.startsWith("#") || line.trim().length() == 0) // Ignore comment characters
continue;
ArrayList<String> lineItems = ListUtilities.CreateStringList(line.split(";"));
String description = lineItems.get(0);
String learnerClassName = lineItems.get(1);
String commandTemplate = (lineItems.size() > 2) ? lineItems.get(2) : "";
LearnerConfigMap.put(
description, new LearnerConfig(description, learnerClassName, commandTemplate));
}
}
/**
* The purpose of this method is to parse the algorithms that have been defined in the algorithm
* configuration files.
*
* @throws Exception
*/
public static void ParseAlgorithms() throws Exception {
for (String line : FileUtilities.ReadLinesFromFile(CLASSIFICATION_ALGORITHMS_FILE)) {
if (line.startsWith("#") || line.trim().length() == 0) // Ignore comment characters
continue;
ArrayList<String> lineItems = ListUtilities.CreateStringList(line.split(";"));
String key = lineItems.remove(0);
String learnerKey = lineItems.remove(0);
CheckLearnerKey(key, learnerKey);
ClassificationAlgorithms.put(key, new ClassificationAlgorithm(key, learnerKey, lineItems));
}
for (String line : FileUtilities.ReadLinesFromFile(FEATURE_SELECTION_ALGORITHMS_FILE)) {
if (line.startsWith("#") || line.trim().length() == 0) // Ignore comment characters
continue;
ArrayList<String> lineItems = ListUtilities.CreateStringList(line.split(";"));
String key = lineItems.remove(0);
String learnerKey = lineItems.remove(0);
ArrayList<String> parameters = lineItems;
CheckLearnerKey(key, learnerKey);
FeatureSelectionAlgorithms.put(
key, new FeatureSelectionAlgorithm(key, learnerKey, parameters));
}
// The following algorithms are made available to any experiment and aren't specified in the
// config file
FeatureSelectionAlgorithms.put(
"None", new FeatureSelectionAlgorithm("None", "", new ArrayList<String>()));
FeatureSelectionAlgorithms.put(
"PriorKnowledge",
new FeatureSelectionAlgorithm("PriorKnowledge", "", new ArrayList<String>()));
}
/**
* Executes a series of tasks that are "locked" such that only one thread should execute each.
* These are locked across multiple compute nodes as well.
*
* @param description Name of the series of tasks that will be executed
* @param lockedCallables Tasks that will be executed
* @throws Exception
*/
public static void ExecuteLockTasks(String description, ArrayList<LockedCallable> lockedCallables)
throws Exception {
MultiThreadedTaskHandler taskHandler = new MultiThreadedTaskHandler(description);
// Prepare the tasks to be executed
for (LockedCallable callable : lockedCallables) {
FileUtilities.CreateFileDirectoryIfNotExists(callable.StatusFilePath);
taskHandler.Add(callable);
}
try {
// See if any of the tasks returned a false value and if so, pause then retry
if (ListUtilities.AnyFalse(ListUtilities.CreateBooleanList(taskHandler.Execute()))) {
Pause(description);
ExecuteLockTasks(description, lockedCallables);
}
} catch (Exception ex) {
// If an exception occurred, log it, pause, then try again
Singletons.Log.Exception(ex);
Pause(description);
ExecuteLockTasks(description, lockedCallables);
}
}
/**
* This method saves various description files to the output. These tasks are quick to execute, so
* they are grouped together rather than parallelized.
*
* @throws Exception
*/
public void SaveExperimentDescriptionFiles() throws Exception {
// Save the version file in output so people know which version was used
FileUtilities.CopyFile(
Settings.VERSION_FILE, Settings.GetOutputSettingsDir(false) + Settings.VERSION_FILE);
// Copy the experiment file
FileUtilities.WriteLineToFile(
Settings.GetOutputSettingsDir(false) + "Experiment_Settings.txt",
"# All settings used in this experiment, whether explicitly set or used by default in the absence of an explicit setting");
FileUtilities.AppendTextToFile(
Settings.GetOutputSettingsDir(false) + "Experiment_Settings.txt",
Singletons.Config.toString());
if (Settings.NeedToClassify()) {
// Copy the algorithm configuration files
FileUtilities.CopyFile(
Settings.LEARNER_TEMPLATES_FILE,
Settings.GetOutputSettingsDir(false)
+ new File(Settings.LEARNER_TEMPLATES_FILE).getName());
// Copy the algorithm configuration files
FileUtilities.CopyFile(
Settings.CLASSIFICATION_ALGORITHMS_FILE,
Settings.GetOutputSettingsDir(false)
+ new File(Settings.CLASSIFICATION_ALGORITHMS_FILE).getName());
if (Settings.NeedToSelectFeatures())
FileUtilities.CopyFile(
Settings.FEATURE_SELECTION_ALGORITHMS_FILE,
Settings.GetOutputSettingsDir(false)
+ new File(Settings.FEATURE_SELECTION_ALGORITHMS_FILE).getName());
SaveExcludedTrainingIDInfo();
SaveCrossValidationAssignments();
}
}
/**
* Depending on the settings for the experiment, this method returns a path to where the
* statistics files should be saved.
*
* @return Path to where the statistics files should be saved
* @throws Exception
*/
public static String GetOutputStatisticsDir() throws Exception {
return FileUtilities.CreateDirectoryIfNotExists(Settings.OUTPUT_DIR + "Statistics/");
}