private static void evaluate( String inFileName, String inFileFormat, int candPerSen, int testIndex) throws Exception { // candPerSen: how many candidates are provided per sentence? // (if inFileFormat is nbest, then candPerSen is ignored, since it is variable) // testIndex: which of the candidates (for each sentence) should be tested? // e.g. testIndex=1 means first candidate should be evaluated // testIndex=candPerSen means last candidate should be evaluated if (inFileFormat.equals("plain") && candPerSen < 1) { println("candPerSen must be positive for a file in plain format."); System.exit(30); } if (inFileFormat.equals("plain") && (testIndex < 1 || testIndex > candPerSen)) { println("For the plain format, testIndex must be in [1,candPerSen]"); System.exit(31); } // read the candidates String[] topCand_str = new String[numSentences]; BufferedReader inFile = new BufferedReader(new FileReader(inFileName)); String line, candidate_str; if (inFileFormat.equals("plain")) { for (int i = 0; i < numSentences; ++i) { // skip candidates 1 through testIndex-1 for (int n = 1; n < testIndex; ++n) { line = inFile.readLine(); } // read testIndex'th candidate candidate_str = inFile.readLine(); topCand_str[i] = candidate_str; for (int n = testIndex + 1; n <= candPerSen; ++n) { // skip candidates testIndex+1 through candPerSen-1 line = inFile.readLine(); } } // for (i) } else { // nbest format int i = 0; int n = 1; line = inFile.readLine(); while (line != null && i < numSentences) { /* line format: .* ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val .* */ while (n < candRank) { line = inFile.readLine(); ++n; } // at the moment, line stores the candRank'th candidate (1-indexed) of the i'th sentence // (0-indexed) if (line == null) { println( "Not enough candidates in " + inFileName + " to extract the " + candRank + "'th candidate for each sentence."); println("(Failed to extract one for the " + i + "'th sentence (0-indexed).)"); System.exit(32); } int read_i = Integer.parseInt(line.substring(0, line.indexOf(" |||"))); if (read_i == i) { line = line.substring(line.indexOf("||| ") + 4); // get rid of initial text candidate_str = line.substring(0, line.indexOf(" |||")); topCand_str[i] = candidate_str; if (i < numSentences - 1) { while (read_i == i) { line = inFile.readLine(); read_i = Integer.parseInt(line.substring(0, line.indexOf(" |||"))); } } n = 1; i += 1; } else { println( "Not enough candidates in " + inFileName + " to extract the " + candRank + "'th candidate for each sentence."); println("(Failed to extract one for the " + i + "'th sentence (0-indexed).)"); System.exit(32); } } // while (line != null) if (i != numSentences) { println( "Not enough candidates were found (i = " + i + "; was expecting " + numSentences + ")"); System.exit(33); } } inFile.close(); evalMetric.printDetailedScore(topCand_str, false); if (verbose) { println(""); println("Printing detailed scores for individual sentences..."); for (int i = 0; i < numSentences; ++i) { print("Sentence #" + i + ": "); evalMetric.printDetailedScore(topCand_str[i], i, true); // already prints a \n } } } // void evaluate(...)