private void processExamples(String group, List<Example> examples) {
Evaluation evaluation = new Evaluation();
if (examples.isEmpty()) return;
final String prefix = "iter=0." + group;
Execution.putOutput("group", group);
LogInfo.begin_track_printAll("Processing %s: %s examples", prefix, examples.size());
LogInfo.begin_track("Dumping metadata");
dumpMetadata(group, examples);
LogInfo.end_track();
LogInfo.begin_track("Examples");
for (int e = 0; e < examples.size(); e++) {
Example ex = examples.get(e);
LogInfo.begin_track_printAll("%s: example %s/%s: %s", prefix, e, examples.size(), ex.id);
ex.log();
Execution.putOutput("example", e);
StopWatchSet.begin("Parser.parse");
ParserState state = builder.parser.parse(params, ex, false);
StopWatchSet.end();
out.printf("########## Example %s ##########\n", ex.id);
dumpExample(exampleToLispTree(state));
LogInfo.logs("Current: %s", ex.evaluation.summary());
evaluation.add(ex.evaluation);
LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary());
LogInfo.end_track();
ex.predDerivations.clear(); // To save memory
}
LogInfo.end_track();
LogInfo.logs("Stats for %s: %s", prefix, evaluation.summary());
evaluation.logStats(prefix);
evaluation.putOutput(prefix);
LogInfo.end_track();
}
private void updateConfusionMatrix(
ConfusionMatrix m, Example ex, double compatDecisionThreshold, double probDecisionThreshold) {
List<Derivation> derivations = ex.getPredDerivations();
double[] probs = Derivation.getProbs(derivations, 1.0d);
for (int i = 0; i < derivations.size(); i++) {
Derivation deriv = derivations.get(i);
double gold, pred;
if (compatDecisionThreshold == -1.0d) gold = deriv.getCompatibility();
else gold = (deriv.getCompatibility() > compatDecisionThreshold) ? 1.0d : 0.0d;
if (probDecisionThreshold == -1.0d) pred = probs[i];
else pred = (probs[i] > probDecisionThreshold) ? 1.0d : 0.0d;
m.tp += gold * pred;
m.fn += gold * (1.0d - pred);
m.fp += (1.0d - gold) * pred;
m.tn += (1.0d - gold) * (1.0d - pred);
}
}