public void run() {
long startTime = System.currentTimeMillis();
List<File> goldFiles;
goldFiles = Evaluator.getGoldFiles(gv.getDataDir());
File evalDir = Helper.createDir(gv.getEvalDir());
String evalFileName =
getExperimentID() + "_" + Helper.getUniqueID() + Evaluator.EVAL_FILE_SUFFIX;
File evalFile = new File(evalDir, evalFileName);
Helper.recursivelyRemove(gv.getResultDir());
StringBuffer buf = new StringBuffer("Dataset\t");
for (int i = 0; i < numExpansions; i++) buf.append("E").append(i + 1).append("\t");
buf.setCharAt(buf.length() - 1, '\n');
Helper.writeToFile(evalFile, buf.toString());
int bufStartIndex = buf.length();
double[] maps = new double[numExpansions];
int goldFileCounts = 0;
for (File goldFile : goldFiles) {
String dataName = Evaluator.getDataName(goldFile);
String langID = Evaluator.getLangID(goldFile);
gv.setLangID(langID);
if (!isExpDataset(langID, dataName)) {
log.warn(dataName + " (" + langID + ") is not interesting, skipping...");
continue;
}
evaluator.loadGoldFile(goldFile);
goldFileCounts++;
for (int trial = 0; trial < numTrials; trial++) {
log.info(Helper.repeat('-', 80));
log.info(
"Evaluating trial "
+ (trial + 1)
+ " out of "
+ numTrials
+ " on "
+ dataName
+ " ("
+ langID
+ ")...");
List<EvalResult> evalResults = evaluate(trial);
buf.append(langID).append(".").append(dataName).append("\t");
double map = 0;
for (int expansion = 0; expansion < numExpansions; expansion++) {
if (expansion < evalResults.size()) map = evalResults.get(expansion).meanAvgPrecision;
maps[expansion] += map;
buf.append(map).append("\t");
}
buf.setCharAt(buf.length() - 1, '\n');
Helper.writeToFile(evalFile, buf.substring(bufStartIndex), true);
bufStartIndex = buf.length();
}
}
buf.append("Average\t");
int totalTrials = goldFileCounts * numTrials;
for (int i = 0; i < numExpansions; i++) buf.append(maps[i] / totalTrials).append("\t");
buf.setCharAt(buf.length() - 1, '\n');
log.info("Writing evaluation results to: " + evalFile);
Helper.writeToFile(evalFile, buf.substring(bufStartIndex), true);
mailResults(getExperimentID(), buf.toString());
Helper.printMemoryUsed();
Helper.printElapsedTime(startTime);
}
