public void run() { long startTime = System.currentTimeMillis(); List<File> goldFiles; goldFiles = Evaluator.getGoldFiles(gv.getDataDir()); File evalDir = Helper.createDir(gv.getEvalDir()); String evalFileName = getExperimentID() + "_" + Helper.getUniqueID() + Evaluator.EVAL_FILE_SUFFIX; File evalFile = new File(evalDir, evalFileName); Helper.recursivelyRemove(gv.getResultDir()); StringBuffer buf = new StringBuffer("Dataset\t"); for (int i = 0; i < numExpansions; i++) buf.append("E").append(i + 1).append("\t"); buf.setCharAt(buf.length() - 1, '\n'); Helper.writeToFile(evalFile, buf.toString()); int bufStartIndex = buf.length(); double[] maps = new double[numExpansions]; int goldFileCounts = 0; for (File goldFile : goldFiles) { String dataName = Evaluator.getDataName(goldFile); String langID = Evaluator.getLangID(goldFile); gv.setLangID(langID); if (!isExpDataset(langID, dataName)) { log.warn(dataName + " (" + langID + ") is not interesting, skipping..."); continue; } evaluator.loadGoldFile(goldFile); goldFileCounts++; for (int trial = 0; trial < numTrials; trial++) { log.info(Helper.repeat('-', 80)); log.info( "Evaluating trial " + (trial + 1) + " out of " + numTrials + " on " + dataName + " (" + langID + ")..."); List<EvalResult> evalResults = evaluate(trial); buf.append(langID).append(".").append(dataName).append("\t"); double map = 0; for (int expansion = 0; expansion < numExpansions; expansion++) { if (expansion < evalResults.size()) map = evalResults.get(expansion).meanAvgPrecision; maps[expansion] += map; buf.append(map).append("\t"); } buf.setCharAt(buf.length() - 1, '\n'); Helper.writeToFile(evalFile, buf.substring(bufStartIndex), true); bufStartIndex = buf.length(); } } buf.append("Average\t"); int totalTrials = goldFileCounts * numTrials; for (int i = 0; i < numExpansions; i++) buf.append(maps[i] / totalTrials).append("\t"); buf.setCharAt(buf.length() - 1, '\n'); log.info("Writing evaluation results to: " + evalFile); Helper.writeToFile(evalFile, buf.substring(bufStartIndex), true); mailResults(getExperimentID(), buf.toString()); Helper.printMemoryUsed(); Helper.printElapsedTime(startTime); }