private void processExamples(String group, List<Example> examples) {
Evaluation evaluation = new Evaluation();
if (examples.isEmpty()) return;
final String prefix = "iter=0." + group;
Execution.putOutput("group", group);
LogInfo.begin_track_printAll("Processing %s: %s examples", prefix, examples.size());
LogInfo.begin_track("Dumping metadata");
dumpMetadata(group, examples);
LogInfo.end_track();
LogInfo.begin_track("Examples");
for (int e = 0; e < examples.size(); e++) {
Example ex = examples.get(e);
LogInfo.begin_track_printAll("%s: example %s/%s: %s", prefix, e, examples.size(), ex.id);
ex.log();
Execution.putOutput("example", e);
StopWatchSet.begin("Parser.parse");
ParserState state = builder.parser.parse(params, ex, false);
StopWatchSet.end();
out.printf("########## Example %s ##########\n", ex.id);
dumpExample(exampleToLispTree(state));
LogInfo.logs("Current: %s", ex.evaluation.summary());
evaluation.add(ex.evaluation);
LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary());
LogInfo.end_track();
ex.predDerivations.clear(); // To save memory
}
LogInfo.end_track();
LogInfo.logs("Stats for %s: %s", prefix, evaluation.summary());
evaluation.logStats(prefix);
evaluation.putOutput(prefix);
LogInfo.end_track();
}
private void dumpMetadata(String group, List<Example> examples) {
LispTree tree = LispTree.proto.newList();
tree.addChild("metadata");
tree.addChild(LispTree.proto.newList("group", group));
tree.addChild(LispTree.proto.newList("size", "" + examples.size()));
tree.print(out);
out.println();
}
@Override
public void run() {
builder = new Builder();
builder.build();
dataset = new Dataset();
dataset.read();
params = new Params();
for (String group : dataset.groups()) {
String filename = Execution.getFile("dumped-" + group + ".gz");
out = IOUtils.openOutHard(filename);
processExamples(group, dataset.examples(group));
out.close();
LogInfo.logs("Finished dumping to %s", filename);
StopWatchSet.logStats();
}
}
private void dumpExample(LispTree tree) {
out.println("(example");
for (LispTree subtree : tree.children.subList(1, tree.children.size())) {
if (!subtree.isLeaf() && "derivations".equals(subtree.children.get(0).value)) {
if (subtree.children.size() == 1) {
out.println(" (derivations)");
} else {
out.println(" (derivations");
for (LispTree derivation : subtree.children.subList(1, subtree.children.size())) {
out.write(" ");
derivation.print(Integer.MAX_VALUE, Integer.MAX_VALUE, out);
out.write("\n");
}
out.println(" )");
}
} else {
out.write(" ");
subtree.print(Integer.MAX_VALUE, Integer.MAX_VALUE, out);
out.write("\n");
}
}
out.println(")");
}