public void train(List<Tree<String>> trainTrees) {
// TODO: before you generate your grammar, the training trees
// need to be binarized so that rules are at most binary
// Binarize the tree.
List<Tree<String>> binarizedTrees = new ArrayList<Tree<String>>();
for (Tree<String> trainTree : trainTrees) {
Tree<String> newTree = TreeAnnotations.annotateTree(trainTree);
binarizedTrees.add(newTree);
}
lexicon = new Lexicon(binarizedTrees);
grammar = new Grammar(binarizedTrees);
System.out.println("trained!!");
}
public Tree<String> getBestParseOld(List<String> sentence) {
// TODO: This implements the CKY algorithm
CounterMap<String, String> parseScores = new CounterMap<String, String>();
System.out.println(sentence.toString());
// First deal with the lexicons
int index = 0;
int span = 1; // All spans are 1 at the lexicon level
for (String word : sentence) {
for (String tag : lexicon.getAllTags()) {
double score = lexicon.scoreTagging(word, tag);
if (score >= 0.0) { // This lexicon may generate this word
// We use a counter map in order to store the scores for this sentence parse.
parseScores.setCount(index + " " + (index + span), tag, score);
}
}
index = index + 1;
}
// handle unary rules now
HashMap<String, Triplet<Integer, String, String>> backHash =
new HashMap<
String, Triplet<Integer, String, String>>(); // hashmap to store back propation
// System.out.println("Lexicons found");
Boolean added = true;
while (added) {
added = false;
for (index = 0; index < sentence.size(); index++) {
// For each index+ span pair, get the counter.
Counter<String> count = parseScores.getCounter(index + " " + (index + span));
PriorityQueue<String> countAsPQ = count.asPriorityQueue();
while (countAsPQ.hasNext()) {
String entry = countAsPQ.next();
// System.out.println("I am fine here!!");
List<UnaryRule> unaryRules = grammar.getUnaryRulesByChild(entry);
for (UnaryRule rule : unaryRules) {
// These are the unary rules which might give rise to the above preterminal
double prob =
rule.getScore() * parseScores.getCount(index + " " + (index + span), entry);
if (prob > parseScores.getCount(index + " " + (index + span), rule.parent)) {
parseScores.setCount(index + " " + (index + span), rule.parent, prob);
backHash.put(
index + " " + (index + span) + " " + rule.parent,
new Triplet<Integer, String, String>(-1, entry, null));
added = true;
}
}
}
}
}
// System.out.println("Lexicon unaries dealt with");
// Now work with the grammar to produce higher level probabilities
for (span = 2; span <= sentence.size(); span++) {
for (int begin = 0; begin <= (sentence.size() - span); begin++) {
int end = begin + span;
for (int split = begin + 1; split <= end - 1; split++) {
Counter<String> countLeft = parseScores.getCounter(begin + " " + split);
Counter<String> countRight = parseScores.getCounter(split + " " + end);
// List<BinaryRule> leftRules= new ArrayList<BinaryRule>();
HashMap<Integer, BinaryRule> leftMap = new HashMap<Integer, BinaryRule>();
// List<BinaryRule> rightRules=new ArrayList<BinaryRule>();
HashMap<Integer, BinaryRule> rightMap = new HashMap<Integer, BinaryRule>();
for (String entry : countLeft.keySet()) {
for (BinaryRule rule : grammar.getBinaryRulesByLeftChild(entry)) {
if (!leftMap.containsKey(rule.hashCode())) {
leftMap.put(rule.hashCode(), rule);
}
}
}
for (String entry : countRight.keySet()) {
for (BinaryRule rule : grammar.getBinaryRulesByRightChild(entry)) {
if (!rightMap.containsKey(rule.hashCode())) {
rightMap.put(rule.hashCode(), rule);
}
}
}
// System.out.println("About to enter the rules loops");
for (Integer ruleHash : leftMap.keySet()) {
if (rightMap.containsKey(ruleHash)) {
BinaryRule ruleRight = rightMap.get(ruleHash);
double prob =
ruleRight.getScore()
* parseScores.getCount(begin + " " + split, ruleRight.leftChild)
* parseScores.getCount(split + " " + end, ruleRight.rightChild);
// System.out.println(begin+" "+ end +" "+ ruleRight.parent+ " "+ prob);
if (prob > parseScores.getCount(begin + " " + end, ruleRight.parent)) {
// System.out.println(begin+" "+ end +" "+ ruleRight.parent+ " "+ prob);
// System.out.println("parentrule :"+ ruleRight.getParent());
parseScores.setCount(begin + " " + end, ruleRight.getParent(), prob);
backHash.put(
begin + " " + end + " " + ruleRight.parent,
new Triplet<Integer, String, String>(
split, ruleRight.leftChild, ruleRight.rightChild));
}
}
}
// System.out.println("Exited rules loop");
}
// System.out.println("Grammar found for " + begin + " "+ end);
// Now handle unary rules
added = true;
while (added) {
added = false;
Counter<String> count = parseScores.getCounter(begin + " " + end);
PriorityQueue<String> countAsPriorityQueue = count.asPriorityQueue();
while (countAsPriorityQueue.hasNext()) {
String entry = countAsPriorityQueue.next();
List<UnaryRule> unaryRules = grammar.getUnaryRulesByChild(entry);
for (UnaryRule rule : unaryRules) {
double prob = rule.getScore() * parseScores.getCount(begin + " " + (end), entry);
if (prob > parseScores.getCount(begin + " " + (end), rule.parent)) {
parseScores.setCount(begin + " " + (end), rule.parent, prob);
backHash.put(
begin + " " + (end) + " " + rule.parent,
new Triplet<Integer, String, String>(-1, entry, null));
added = true;
}
}
}
}
// System.out.println("Unaries dealt for " + begin + " "+ end);
}
}
// Create and return the parse tree
Tree<String> parseTree = new Tree<String>("null");
// System.out.println(parseScores.getCounter(0+" "+sentence.size()).toString());
String parent = parseScores.getCounter(0 + " " + sentence.size()).argMax();
if (parent == null) {
System.out.println(parseScores.getCounter(0 + " " + sentence.size()).toString());
System.out.println("THIS IS WEIRD");
}
parent = "ROOT";
parseTree = getParseTreeOld(sentence, backHash, 0, sentence.size(), parent);
// System.out.println("PARSE SCORES");
// System.out.println(parseScores.toString());
// System.out.println("BACK HASH");
// System.out.println(backHash.toString());
// parseTree = addRoot(parseTree);
// System.out.println(parseTree.toString());
// return parseTree;
return TreeAnnotations.unAnnotateTree(parseTree);
}
public Tree<String> getBestParse(List<String> sentence) {
// This implements the CKY algorithm
int nEntries = sentence.size();
// hashmap to store back rules
HashMap<Triplet<Integer, Integer, String>, Triplet<Integer, String, String>> backHash =
new HashMap<Triplet<Integer, Integer, String>, Triplet<Integer, String, String>>();
// more efficient access with arrays, but must cast each time :(
@SuppressWarnings("unchecked")
Counter<String>[][] parseScores = (Counter<String>[][]) (new Counter[nEntries][nEntries]);
for (int i = 0; i < nEntries; i++) {
for (int j = 0; j < nEntries; j++) {
parseScores[i][j] = new Counter<String>();
}
}
System.out.println(sentence.toString());
// First deal with the lexicons
int index = 0;
int span = 1; // All spans are 1 at the lexicon level
for (String word : sentence) {
for (String tag : lexicon.getAllTags()) {
double score = lexicon.scoreTagging(word, tag);
if (score >= 0.0) { // This lexicon may generate this word
// We use a counter map in order to store the scores for this sentence parse.
parseScores[index][index + span - 1].setCount(tag, score);
}
}
index = index + 1;
}
// handle unary rules now
// System.out.println("Lexicons found");
boolean added = true;
while (added) {
added = false;
for (index = 0; index < sentence.size(); index++) {
// For each index+ span pair, get the counter.
Counter<String> count = parseScores[index][index + span - 1];
PriorityQueue<String> countAsPQ = count.asPriorityQueue();
while (countAsPQ.hasNext()) {
String entry = countAsPQ.next();
// System.out.println("I am fine here!!");
List<UnaryRule> unaryRules = grammar.getUnaryRulesByChild(entry);
for (UnaryRule rule : unaryRules) {
// These are the unary rules which might give rise to the above preterminal
double prob = rule.getScore() * parseScores[index][index + span - 1].getCount(entry);
if (prob > parseScores[index][index + span - 1].getCount(rule.parent)) {
parseScores[index][index + span - 1].setCount(rule.parent, prob);
backHash.put(
new Triplet<Integer, Integer, String>(index, index + span, rule.parent),
new Triplet<Integer, String, String>(-1, entry, null));
added = true;
}
}
}
}
}
// System.out.println("Lexicon unaries dealt with");
// Now work with the grammar to produce higher level probabilities
for (span = 2; span <= sentence.size(); span++) {
for (int begin = 0; begin <= (sentence.size() - span); begin++) {
int end = begin + span;
for (int split = begin + 1; split <= end - 1; split++) {
Counter<String> countLeft = parseScores[begin][split - 1];
Counter<String> countRight = parseScores[split][end - 1];
// List<BinaryRule> leftRules= new ArrayList<BinaryRule>();
HashMap<Integer, BinaryRule> leftMap = new HashMap<Integer, BinaryRule>();
// List<BinaryRule> rightRules=new ArrayList<BinaryRule>();
HashMap<Integer, BinaryRule> rightMap = new HashMap<Integer, BinaryRule>();
for (String entry : countLeft.keySet()) {
for (BinaryRule rule : grammar.getBinaryRulesByLeftChild(entry)) {
if (!leftMap.containsKey(rule.hashCode())) {
leftMap.put(rule.hashCode(), rule);
}
}
}
for (String entry : countRight.keySet()) {
for (BinaryRule rule : grammar.getBinaryRulesByRightChild(entry)) {
if (!rightMap.containsKey(rule.hashCode())) {
rightMap.put(rule.hashCode(), rule);
}
}
}
// System.out.println("About to enter the rules loops");
for (Integer ruleHash : leftMap.keySet()) {
if (rightMap.containsKey(ruleHash)) {
BinaryRule ruleRight = rightMap.get(ruleHash);
double prob =
ruleRight.getScore()
* parseScores[begin][split - 1].getCount(ruleRight.leftChild)
* parseScores[split][end - 1].getCount(ruleRight.rightChild);
// System.out.println(begin+" "+ end +" "+ ruleRight.parent+ " "+ prob);
if (prob > parseScores[begin][end - 1].getCount(ruleRight.parent)) {
// System.out.println(begin+" "+ end +" "+ ruleRight.parent+ " "+ prob);
// System.out.println("parentrule :"+ ruleRight.getParent());
parseScores[begin][end - 1].setCount(ruleRight.getParent(), prob);
backHash.put(
new Triplet<Integer, Integer, String>(begin, end, ruleRight.parent),
new Triplet<Integer, String, String>(
split, ruleRight.leftChild, ruleRight.rightChild));
}
}
}
// System.out.println("Exited rules loop");
}
// System.out.println("Grammar found for " + begin + " "+ end);
// Now handle unary rules
added = true;
while (added) {
added = false;
Counter<String> count = parseScores[begin][end - 1];
PriorityQueue<String> countAsPriorityQueue = count.asPriorityQueue();
while (countAsPriorityQueue.hasNext()) {
String entry = countAsPriorityQueue.next();
List<UnaryRule> unaryRules = grammar.getUnaryRulesByChild(entry);
for (UnaryRule rule : unaryRules) {
double prob = rule.getScore() * parseScores[begin][end - 1].getCount(entry);
if (prob > parseScores[begin][end - 1].getCount(rule.parent)) {
parseScores[begin][end - 1].setCount(rule.parent, prob);
backHash.put(
new Triplet<Integer, Integer, String>(begin, end, rule.parent),
new Triplet<Integer, String, String>(-1, entry, null));
added = true;
}
}
}
}
// System.out.println("Unaries dealt for " + begin + " "+ end);
}
}
// Create and return the parse tree
Tree<String> parseTree = new Tree<String>("null");
// System.out.println(parseScores.getCounter(0+" "+sentence.size()).toString());
// Pick the argmax
String parent = parseScores[0][nEntries - 1].argMax();
// Or pick root. This second one is preferred since sentences are meant to have ROOT as their
// root node.
parent = "ROOT";
parseTree = getParseTree(sentence, backHash, 0, sentence.size(), parent);
// System.out.println("PARSE SCORES");
// System.out.println(parseScores.toString());
// System.out.println("BACK HASH");
// System.out.println(backHash.toString());
// parseTree = addRoot(parseTree);
// System.out.println(parseTree.toString());
// return parseTree;
return TreeAnnotations.unAnnotateTree(parseTree);
}