public String toString() {
StringBuilder sb = new StringBuilder();
List<String> ruleStrings = new ArrayList<String>();
for (String leftChild : binaryRulesByLeftChild.keySet()) {
for (BinaryRule binaryRule : getBinaryRulesByLeftChild(leftChild)) {
ruleStrings.add(binaryRule.toString());
}
}
for (String child : unaryRulesByChild.keySet()) {
for (UnaryRule unaryRule : getUnaryRulesByChild(child)) {
ruleStrings.add(unaryRule.toString());
}
}
for (String ruleString : CollectionUtils.sort(ruleStrings)) {
sb.append(ruleString);
sb.append("\n");
}
return sb.toString();
}
/* A builds PCFG using the observed counts of binary and unary
* productions in the training trees to estimate the probabilities
* for those rules.
*/
public Grammar(List<Tree<String>> trainTrees) {
Counter<UnaryRule> unaryRuleCounter = new Counter<UnaryRule>();
Counter<BinaryRule> binaryRuleCounter = new Counter<BinaryRule>();
Counter<String> symbolCounter = new Counter<String>();
for (Tree<String> trainTree : trainTrees) {
tallyTree(trainTree, symbolCounter, unaryRuleCounter, binaryRuleCounter);
}
for (UnaryRule unaryRule : unaryRuleCounter.keySet()) {
double unaryProbability =
unaryRuleCounter.getCount(unaryRule) / symbolCounter.getCount(unaryRule.getParent());
unaryRule.setScore(unaryProbability);
addUnary(unaryRule);
}
for (BinaryRule binaryRule : binaryRuleCounter.keySet()) {
double binaryProbability =
binaryRuleCounter.getCount(binaryRule) / symbolCounter.getCount(binaryRule.getParent());
binaryRule.setScore(binaryProbability);
addBinary(binaryRule);
}
}
private void addBinary(BinaryRule binaryRule) {
CollectionUtils.addToValueList(binaryRulesByLeftChild, binaryRule.getLeftChild(), binaryRule);
CollectionUtils.addToValueList(
binaryRulesByRightChild, binaryRule.getRightChild(), binaryRule);
}
public Tree<String> getBestParse(List<String> sentence) {
// TODO: implement this method
int n = sentence.size();
// System.out.println("getBestParse: n=" + n);
List<List<Map<Object, Double>>> scores = new ArrayList<List<Map<Object, Double>>>(n + 1);
for (int i = 0; i < n + 1; i++) {
List<Map<Object, Double>> row = new ArrayList<Map<Object, Double>>(n + 1);
for (int j = 0; j < n + 1; j++) {
row.add(new HashMap<Object, Double>());
}
scores.add(row);
}
List<List<Map<Object, Triplet<Integer, Object, Object>>>> backs =
new ArrayList<List<Map<Object, Triplet<Integer, Object, Object>>>>(n + 1);
for (int i = 0; i < n + 1; i++) {
List<Map<Object, Triplet<Integer, Object, Object>>> row =
new ArrayList<Map<Object, Triplet<Integer, Object, Object>>>(n + 1);
for (int j = 0; j < n + 1; j++) {
row.add(new HashMap<Object, Triplet<Integer, Object, Object>>());
}
backs.add(row);
}
/*
System.out.println("scores=" + scores.size() + "x" + scores.get(0).size());
System.out.println("backs=" + backs.size() + "x" + backs.get(0).size());
printChart(scores, backs, "scores");
*/
// First the Lexicon
for (int i = 0; i < n; i++) {
String word = sentence.get(i);
for (String tag : lexicon.getAllTags()) {
UnaryRule A = new UnaryRule(tag, word);
A.setScore(Math.log(lexicon.scoreTagging(word, tag)));
scores.get(i).get(i + 1).put(A, A.getScore());
backs.get(i).get(i + 1).put(A, null);
}
// System.out.println("Starting unaries: i=" + i + ",n=" + n );
// Handle unaries
boolean added = true;
while (added) {
added = false;
Map<Object, Double> A_scores = scores.get(i).get(i + 1);
// Don't modify the dict we are iterating
List<Object> A_keys = copyKeys(A_scores);
// for (int j = 0; j < 5 && j < A_keys.size(); j++) {
// System.out.print("," + j + "=" + A_scores.get(A_keys.get(j)));
// }
for (Object oB : A_keys) {
UnaryRule B = (UnaryRule) oB;
for (UnaryRule A : grammar.getUnaryRulesByChild(B.getParent())) {
double prob = Math.log(A.getScore()) + A_scores.get(B);
if (prob > -1000.0) {
if (!A_scores.containsKey(A) || prob > A_scores.get(A)) {
// System.out.print(" *A=" + A + ", B=" + B);
// System.out.print(", prob=" + prob);
// System.out.println(", A_scores.get(A)=" + A_scores.get(A));
A_scores.put(A, prob);
backs.get(i).get(i + 1).put(A, new Triplet<Integer, Object, Object>(-1, B, null));
added = true;
}
// System.out.println(", added=" + added);
}
}
}
// System.out.println(", A_scores=" + A_scores.size() + ", added=" + added);
}
}
// printChart(scores, backs, "scores with Lexicon");
// Do higher layers
// Naming is based on rules: A -> B,C
long startTime = new Date().getTime();
for (int span = 2; span < n + 1; span++) {
for (int begin = 0; begin < n + 1 - span; begin++) {
int end = begin + span;
Map<Object, Double> A_scores = scores.get(begin).get(end);
Map<Object, Triplet<Integer, Object, Object>> A_backs = backs.get(begin).get(end);
for (int split = begin + 1; split < end; split++) {
Map<Object, Double> B_scores = scores.get(begin).get(split);
Map<Object, Double> C_scores = scores.get(split).get(end);
List<Object> B_list = new ArrayList<Object>(B_scores.keySet());
List<Object> C_list = new ArrayList<Object>(C_scores.keySet());
// This is a key optimization. !@#$
// It avoids a B_list.size() x C_list.size() search in the for (Object B : B_list) loop
Map<String, List<Object>> C_map = new HashMap<String, List<Object>>();
for (Object C : C_list) {
String parent = getParent(C);
if (!C_map.containsKey(parent)) {
C_map.put(parent, new ArrayList<Object>());
}
C_map.get(parent).add(C);
}
for (Object B : B_list) {
for (BinaryRule A : grammar.getBinaryRulesByLeftChild(getParent(B))) {
if (C_map.containsKey(A.getRightChild())) {
for (Object C : C_map.get(A.getRightChild())) {
// We now have A which has B as left child and C as right child
double prob = Math.log(A.getScore()) + B_scores.get(B) + C_scores.get(C);
if (!A_scores.containsKey(A) || prob > A_scores.get(A)) {
A_scores.put(A, prob);
A_backs.put(A, new Triplet<Integer, Object, Object>(split, B, C));
}
}
}
}
}
}
// Handle unaries: A -> B
boolean added = true;
while (added) {
added = false;
// Don't modify the dict we are iterating
List<Object> A_keys = copyKeys(A_scores);
for (Object oB : A_keys) {
for (UnaryRule A : grammar.getUnaryRulesByChild(getParent(oB))) {
double prob = Math.log(A.getScore()) + A_scores.get(oB);
if (!A_scores.containsKey(A) || prob > A_scores.get(A)) {
A_scores.put(A, prob);
A_backs.put(A, new Triplet<Integer, Object, Object>(-1, oB, null));
added = true;
}
}
}
}
}
}
// printChart(scores, backs, "scores with Lexicon and Grammar");
Map<Object, Double> topOfChart = scores.get(0).get(n);
System.out.println("topOfChart: " + topOfChart.size());
/*
for (Object o: topOfChart.keySet()) {
System.out.println("o=" + o + ", score=" + topOfChart.getCount(o));
}
*/
// All parses have "ROOT" at top of tree
Object bestKey = null;
Object secondBestKey = null;
double bestScore = Double.NEGATIVE_INFINITY;
double secondBestScore = Double.NEGATIVE_INFINITY;
for (Object key : topOfChart.keySet()) {
double score = topOfChart.get(key);
if (score >= secondBestScore || secondBestKey == null) {
secondBestKey = key;
secondBestScore = score;
}
if ("ROOT".equals(getParent(key)) && (score >= bestScore || bestKey == null)) {
bestKey = key;
bestScore = score;
}
}
if (bestKey == null) {
bestKey = secondBestKey;
System.out.println("secondBestKey=" + secondBestKey);
}
if (bestKey == null) {
for (Object key : topOfChart.keySet()) {
System.out.println("val=" + topOfChart.get(key) + ", key=" + key);
}
}
System.out.println("bestKey=" + bestKey + ", log(prob)=" + topOfChart.get(bestKey));
Tree<String> result = makeTree(backs, 0, n, bestKey);
if (!"ROOT".equals(result.getLabel())) {
List<Tree<String>> children = new ArrayList<Tree<String>>();
children.add(result);
result = new Tree<String>("ROOT", children); // !@#$
}
/*
System.out.println("==================================================");
System.out.println(result);
System.out.println("====================^^^^^^========================");
*/
return TreeAnnotations.unAnnotateTree(result);
}