public static void main(String[] args) throws Exception { if (args.length == 0) { printUsage(args.length); System.exit(0); } else { processArgsAndInitialize(args); } // non-specified args will be set to default values in processArgsAndInitialize if (candFileFormat.equals("plain")) { println("Evaluating candidate translations in plain file " + candFileName + "..."); evaluateCands_plain(candFileName); } else if (candFileFormat.equals("nbest")) { println( "Evaluating set of " + candRank + "'th candidate translations from " + candFileName + "..."); evaluateCands_nbest(candFileName, candRank); } println(""); if (evaluateRefs) { // evaluate the references themselves; useful if developing a new evaluation metric println(""); println("PERFORMING SANITY CHECK:"); println("------------------------"); println(""); println( "This metric's scores range from " + evalMetric.worstPossibleScore() + " (worst) to " + evalMetric.bestPossibleScore() + " (best)."); for (int r = 1; r <= refsPerSen; ++r) { println(""); println("(*) Evaluating reference set " + r + ":"); println(""); evaluateRefSet(r); println(""); } } System.exit(0); } // main(String[] args)
@Override public List<ScoredFeaturizedTranslation<TK, FV>> maximize(NBestListContainer<TK, FV> nbest) { List<ScoredFeaturizedTranslation<TK, FV>> selected = greedyMetrixMax.maximize(nbest); List<List<ScoredFeaturizedTranslation<TK, FV>>> nbestLists = nbest.nbestLists(); IncrementalEvaluationMetric<TK, FV> incrementalMetric = metric.getIncrementalMetric(); IncrementalEvaluationMetric<TK, FV> incrementalSubMetric = (subMetric != null ? subMetric.getIncrementalMetric() : null); for (ScoredFeaturizedTranslation<TK, FV> featurizedTranslation : selected) { incrementalMetric.add(featurizedTranslation); if (incrementalSubMetric != null) incrementalSubMetric.add(featurizedTranslation); } int nbestListsSize = nbestLists.size(); int iter = 0; for (int changes = nbestListsSize; changes != 0 && iter < 25; iter++) { // XXX changes = 0; for (int i = 0; i < nbestListsSize; i++) { List<ScoredFeaturizedTranslation<TK, FV>> nbestList = nbestLists.get(i); ScoredFeaturizedTranslation<TK, FV> bestFTrans = null; double bestScore = Double.NaN; double bestScoreSub = Double.NaN; // int tI = -1; for (ScoredFeaturizedTranslation<TK, FV> ftrans : nbestList) { // tI++; incrementalMetric.replace(i, ftrans); if (subMetric != null) { assert (incrementalSubMetric != null); incrementalSubMetric.replace(i, ftrans); } double score = incrementalMetric.score(); // System.err.printf("bestScore(%d): %f score: %f \n", bestI, // bestScore, score); if (bestScore != bestScore || bestScore < score) { bestFTrans = ftrans; bestScore = score; if (subMetric != null) { bestScoreSub = incrementalSubMetric.score(); } } else if (bestScore == score && subMetric != null) { double subMetricScore = incrementalSubMetric.score(); if (bestScoreSub < subMetricScore) { bestScoreSub = subMetricScore; bestFTrans = ftrans; } } } incrementalMetric.replace(i, bestFTrans); if (incrementalSubMetric != null) incrementalSubMetric.replace(i, bestFTrans); if (selected.get(i) != bestFTrans) { changes++; selected.set(i, bestFTrans); } } if (DEBUG) { /* * IncrementalEvaluationMetric<TK,FV> iMetric = * metric.getIncrementalMetric(); for (ScoredFeaturizedTranslation<TK, * FV> trans : selected) { iMetric.add(trans); } if (iMetric.score() != * incrementalMetric.score()) { * * /*System.err.printf("cnt: %d null cnt: %d\n", * ((TERMetric.TERIncrementalMetric)incrementalMetric).cnt, * ((TERMetric.TERIncrementalMetric)incrementalMetric).nullCnt); * System.err.printf("cnt: %d null cnt: %d\n", * ((TERMetric.TERIncrementalMetric)iMetric).cnt, * ((TERMetric.TERIncrementalMetric)iMetric).nullCnt); / throw new * RuntimeException(String.format("%f!=%f\n", incrementalMetric.score(), * iMetric.score())); } */ System.err.printf( "%d: score: %.5f changes: %d\n", iter, incrementalMetric.score(), changes); /* * for (ScoredFeaturizedTranslation<TK,FV> t : selected) { * System.err.printf("%s\n", (t == null ? t : t.translation)); } */ } } return selected; }
private static void processArgsAndInitialize(String[] args) throws Exception { EvaluationMetric.set_knownMetrics(); // set default values candFileName = "candidates.txt"; candFileFormat = "plain"; candRank = 1; refFileName = "references.txt"; refsPerSen = 1; metricName = "BLEU"; metricOptions = new String[2]; metricOptions[0] = "4"; metricOptions[1] = "closest"; evaluateRefs = false; verbose = false; int i = 0; while (i < args.length) { String option = args[i]; if (option.equals("-cand")) { candFileName = args[i + 1]; } else if (option.equals("-format")) { candFileFormat = args[i + 1]; if (!candFileFormat.equals("plain") && !candFileFormat.equals("nbest")) { println("candFileFormat must be either plain or nbest."); System.exit(10); } } else if (option.equals("-rank")) { candRank = Integer.parseInt(args[i + 1]); if (refsPerSen < 1) { println("Argument for -rank must be positive."); System.exit(10); } } else if (option.equals("-ref")) { refFileName = args[i + 1]; } else if (option.equals("-rps")) { refsPerSen = Integer.parseInt(args[i + 1]); if (refsPerSen < 1) { println("refsPerSen must be positive."); System.exit(10); } } else if (option.equals("-m")) { metricName = args[i + 1]; if (EvaluationMetric.knownMetricName(metricName)) { int optionCount = EvaluationMetric.metricOptionCount(metricName); metricOptions = new String[optionCount]; for (int opt = 0; opt < optionCount; ++opt) { metricOptions[opt] = args[i + opt + 2]; } i += optionCount; } else { println("Unknown metric name " + metricName + "."); System.exit(10); } } /* else if (option.equals("-m")) { metricName = args[i+1]; if (!EvaluationMetric.knownMetricName(metricName)) { println("Unknown metric name " + metricName + "."); System.exit(10); } if (metricName.equals("BLEU")) { metricOptions = new String[2]; metricOptions[0] = args[i+2]; metricOptions[1] = args[i+3]; i += 2; } } */ else if (option.equals("-evr")) { int evr = Integer.parseInt(args[i + 1]); if (evr == 1) evaluateRefs = true; else if (evr == 0) evaluateRefs = false; else { println("evalRefs must be either 0 or 1."); System.exit(10); } } else if (option.equals("-v")) { int v = Integer.parseInt(args[i + 1]); if (v == 1) verbose = true; else if (v == 0) verbose = false; else { println("verbose must be either 0 or 1."); System.exit(10); } } else { println("Unknown option " + option); System.exit(10); } i += 2; } // while (i) if (refsPerSen > 1) { // the provided refFileName might be a prefix File dummy = new File(refFileName); if (!dummy.exists()) { refFileName = createUnifiedRefFile(refFileName, refsPerSen); } } else { checkFile(refFileName); } // initialize numSentences = countLines(refFileName) / refsPerSen; // read in reference sentences refSentences = new String[numSentences][refsPerSen]; BufferedReader inFile_refs = new BufferedReader(new FileReader(refFileName)); String line; for (i = 0; i < numSentences; ++i) { for (int r = 0; r < refsPerSen; ++r) { // read the rth reference translation for the ith sentence refSentences[i][r] = inFile_refs.readLine(); } } inFile_refs.close(); // set static data members for the EvaluationMetric class EvaluationMetric.set_numSentences(numSentences); EvaluationMetric.set_refsPerSen(refsPerSen); EvaluationMetric.set_refSentences(refSentences); // do necessary initialization for the evaluation metric evalMetric = EvaluationMetric.getMetric(metricName, metricOptions); println("Processing " + numSentences + " sentences..."); } // processArgsAndInitialize(String[] args)
private static void evaluate( String inFileName, String inFileFormat, int candPerSen, int testIndex) throws Exception { // candPerSen: how many candidates are provided per sentence? // (if inFileFormat is nbest, then candPerSen is ignored, since it is variable) // testIndex: which of the candidates (for each sentence) should be tested? // e.g. testIndex=1 means first candidate should be evaluated // testIndex=candPerSen means last candidate should be evaluated if (inFileFormat.equals("plain") && candPerSen < 1) { println("candPerSen must be positive for a file in plain format."); System.exit(30); } if (inFileFormat.equals("plain") && (testIndex < 1 || testIndex > candPerSen)) { println("For the plain format, testIndex must be in [1,candPerSen]"); System.exit(31); } // read the candidates String[] topCand_str = new String[numSentences]; BufferedReader inFile = new BufferedReader(new FileReader(inFileName)); String line, candidate_str; if (inFileFormat.equals("plain")) { for (int i = 0; i < numSentences; ++i) { // skip candidates 1 through testIndex-1 for (int n = 1; n < testIndex; ++n) { line = inFile.readLine(); } // read testIndex'th candidate candidate_str = inFile.readLine(); topCand_str[i] = candidate_str; for (int n = testIndex + 1; n <= candPerSen; ++n) { // skip candidates testIndex+1 through candPerSen-1 line = inFile.readLine(); } } // for (i) } else { // nbest format int i = 0; int n = 1; line = inFile.readLine(); while (line != null && i < numSentences) { /* line format: .* ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val .* */ while (n < candRank) { line = inFile.readLine(); ++n; } // at the moment, line stores the candRank'th candidate (1-indexed) of the i'th sentence // (0-indexed) if (line == null) { println( "Not enough candidates in " + inFileName + " to extract the " + candRank + "'th candidate for each sentence."); println("(Failed to extract one for the " + i + "'th sentence (0-indexed).)"); System.exit(32); } int read_i = Integer.parseInt(line.substring(0, line.indexOf(" |||"))); if (read_i == i) { line = line.substring(line.indexOf("||| ") + 4); // get rid of initial text candidate_str = line.substring(0, line.indexOf(" |||")); topCand_str[i] = candidate_str; if (i < numSentences - 1) { while (read_i == i) { line = inFile.readLine(); read_i = Integer.parseInt(line.substring(0, line.indexOf(" |||"))); } } n = 1; i += 1; } else { println( "Not enough candidates in " + inFileName + " to extract the " + candRank + "'th candidate for each sentence."); println("(Failed to extract one for the " + i + "'th sentence (0-indexed).)"); System.exit(32); } } // while (line != null) if (i != numSentences) { println( "Not enough candidates were found (i = " + i + "; was expecting " + numSentences + ")"); System.exit(33); } } inFile.close(); evalMetric.printDetailedScore(topCand_str, false); if (verbose) { println(""); println("Printing detailed scores for individual sentences..."); for (int i = 0; i < numSentences; ++i) { print("Sentence #" + i + ": "); evalMetric.printDetailedScore(topCand_str[i], i, true); // already prints a \n } } } // void evaluate(...)