private static void evaluate(
String inFileName, String inFileFormat, int candPerSen, int testIndex) throws Exception {
// candPerSen: how many candidates are provided per sentence?
// (if inFileFormat is nbest, then candPerSen is ignored, since it is variable)
// testIndex: which of the candidates (for each sentence) should be tested?
// e.g. testIndex=1 means first candidate should be evaluated
// testIndex=candPerSen means last candidate should be evaluated
if (inFileFormat.equals("plain") && candPerSen < 1) {
println("candPerSen must be positive for a file in plain format.");
System.exit(30);
}
if (inFileFormat.equals("plain") && (testIndex < 1 || testIndex > candPerSen)) {
println("For the plain format, testIndex must be in [1,candPerSen]");
System.exit(31);
}
// read the candidates
String[] topCand_str = new String[numSentences];
BufferedReader inFile = new BufferedReader(new FileReader(inFileName));
String line, candidate_str;
if (inFileFormat.equals("plain")) {
for (int i = 0; i < numSentences; ++i) {
// skip candidates 1 through testIndex-1
for (int n = 1; n < testIndex; ++n) {
line = inFile.readLine();
}
// read testIndex'th candidate
candidate_str = inFile.readLine();
topCand_str[i] = candidate_str;
for (int n = testIndex + 1; n <= candPerSen; ++n) {
// skip candidates testIndex+1 through candPerSen-1
line = inFile.readLine();
}
} // for (i)
} else { // nbest format
int i = 0;
int n = 1;
line = inFile.readLine();
while (line != null && i < numSentences) {
/*
line format:
.* ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val .*
*/
while (n < candRank) {
line = inFile.readLine();
++n;
}
// at the moment, line stores the candRank'th candidate (1-indexed) of the i'th sentence
// (0-indexed)
if (line == null) {
println(
"Not enough candidates in "
+ inFileName
+ " to extract the "
+ candRank
+ "'th candidate for each sentence.");
println("(Failed to extract one for the " + i + "'th sentence (0-indexed).)");
System.exit(32);
}
int read_i = Integer.parseInt(line.substring(0, line.indexOf(" |||")));
if (read_i == i) {
line = line.substring(line.indexOf("||| ") + 4); // get rid of initial text
candidate_str = line.substring(0, line.indexOf(" |||"));
topCand_str[i] = candidate_str;
if (i < numSentences - 1) {
while (read_i == i) {
line = inFile.readLine();
read_i = Integer.parseInt(line.substring(0, line.indexOf(" |||")));
}
}
n = 1;
i += 1;
} else {
println(
"Not enough candidates in "
+ inFileName
+ " to extract the "
+ candRank
+ "'th candidate for each sentence.");
println("(Failed to extract one for the " + i + "'th sentence (0-indexed).)");
System.exit(32);
}
} // while (line != null)
if (i != numSentences) {
println(
"Not enough candidates were found (i = " + i + "; was expecting " + numSentences + ")");
System.exit(33);
}
}
inFile.close();
evalMetric.printDetailedScore(topCand_str, false);
if (verbose) {
println("");
println("Printing detailed scores for individual sentences...");
for (int i = 0; i < numSentences; ++i) {
print("Sentence #" + i + ": ");
evalMetric.printDetailedScore(topCand_str[i], i, true);
// already prints a \n
}
}
} // void evaluate(...)
