public static void main(String[] args) {
QPTreeTransformer transformer = new QPTreeTransformer();
Treebank tb = new MemoryTreebank();
Properties props = StringUtils.argsToProperties(args);
String treeFileName = props.getProperty("treeFile");
if (treeFileName != null) {
try {
TreeReader tr =
new PennTreeReader(
new BufferedReader(new InputStreamReader(new FileInputStream(treeFileName))),
new LabeledScoredTreeFactory());
Tree t;
while ((t = tr.readTree()) != null) {
tb.add(t);
}
} catch (IOException e) {
throw new RuntimeException("File problem: " + e);
}
}
for (Tree t : tb) {
System.out.println("Original tree");
t.pennPrint();
System.out.println();
System.out.println("Tree transformed");
Tree tree = transformer.transformTree(t);
tree.pennPrint();
System.out.println();
System.out.println("----------------------------");
}
}
private static void testTransAndUntrans(
CharacterLevelTagExtender e, Treebank tb, PrintWriter pw) {
for (Tree tree : tb) {
Tree oldTree = tree.treeSkeletonCopy();
e.transformTree(tree);
CharacterLevelTagExtender.untransformTree(tree);
if (!tree.equals(oldTree)) {
pw.println("NOT EQUAL AFTER UNTRANSFORMATION!!!");
pw.println();
oldTree.pennPrint(pw);
pw.println();
tree.pennPrint(pw);
pw.println("------------------");
}
}
}
/** For testing: loads a treebank and prints the trees. */
public static void main(String[] args) {
TreebankLangParserParams tlpp = new ChineseTreebankParserParams();
System.out.println("Default encoding is: " + tlpp.diskTreebank().encoding());
if (args.length < 2) {
printlnErr(
"Usage: edu.stanford.nlp.parser.lexparser.ChineseTreebankParserParams treesPath fileRange");
} else {
Treebank m = tlpp.diskTreebank();
m.loadPath(args[0], new NumberRangesFileFilter(args[1], false));
for (Tree t : m) {
t.pennPrint(tlpp.pw());
}
System.out.println("There were " + m.size() + " trees.");
}
}
/**
* Determine which daughter of the current parse tree is the head. It assumes that the daughters
* already have had their heads determined. Uses special rule for VPheads
*
* @param t The parse tree to examine the daughters of. This is assumed to never be a leaf
* @return The parse tree that is the head
*/
protected Tree determineNonTrivialHead(Tree t, Tree parent) {
String motherCat = tlp.basicCategory(t.label().value());
if (DEBUG) {
System.err.println("My parent is " + parent);
}
// do VPs with auxiliary as special case
if (motherCat.equals("VP") || motherCat.equals("SQ") || motherCat.equals("SINV")) {
Tree[] kids = t.children();
// try to find if there is an auxiliary verb
if (DEBUG) {
System.err.println("Semantic head finder: at VP");
System.err.println("Class is " + t.getClass().getName());
t.pennPrint(System.err);
System.err.println("hasVerbalAuxiliary = " + hasVerbalAuxiliary(kids, verbalAuxiliaries));
}
// looks for auxiliaries
if (hasVerbalAuxiliary(kids, verbalAuxiliaries)) {
// String[] how = new String[] {"left", "VP", "ADJP", "NP"};
// Including NP etc seems okay for copular sentences but is
// problematic for other auxiliaries, like 'he has an answer'
// But maybe doing ADJP is fine!
String[] how = new String[] {"left", "VP", "ADJP"};
Tree pti = traverseLocate(kids, how, false);
if (DEBUG) {
System.err.println("Determined head is: " + pti);
}
if (pti != null) {
return pti;
} else {
// System.err.println("------");
// System.err.println("SemanticHeadFinder failed to reassign head for");
// t.pennPrint(System.err);
// System.err.println("------");
}
}
// looks for copular verbs
if (hasVerbalAuxiliary(kids, copulars) && !isExistential(t, parent) && !isWHQ(t, parent)) {
String[] how;
if (motherCat.equals("SQ")) {
how = new String[] {"right", "VP", "ADJP", "NP", "WHADJP", "WHNP"};
} else {
how = new String[] {"left", "VP", "ADJP", "NP", "WHADJP", "WHNP"};
}
Tree pti = traverseLocate(kids, how, false);
if (DEBUG) {
System.err.println("Determined head is: " + pti);
}
if (pti != null) {
return pti;
} else {
if (DEBUG) {
System.err.println("------");
System.err.println("SemanticHeadFinder failed to reassign head for");
t.pennPrint(System.err);
System.err.println("------");
}
}
}
}
return super.determineNonTrivialHead(t, parent);
}
/**
* Determine which daughter of the current parse tree is the head. It assumes that the daughters
* already have had their heads determined. Uses special rule for VP heads
*
* @param t The parse tree to examine the daughters of. This is assumed to never be a leaf
* @return The parse tree that is the head
*/
@Override
protected Tree determineNonTrivialHead(Tree t, Tree parent) {
String motherCat = tlp.basicCategory(t.label().value());
if (DEBUG) {
System.err.println("At " + motherCat + ", my parent is " + parent);
}
// do VPs with auxiliary as special case
if ((motherCat.equals("VP") || motherCat.equals("SQ") || motherCat.equals("SINV"))) {
Tree[] kids = t.children();
// try to find if there is an auxiliary verb
if (DEBUG) {
System.err.println("Semantic head finder: at VP");
System.err.println("Class is " + t.getClass().getName());
t.pennPrint(System.err);
// System.err.println("hasVerbalAuxiliary = " + hasVerbalAuxiliary(kids,
// verbalAuxiliaries));
}
// looks for auxiliaries
if (hasVerbalAuxiliary(kids, verbalAuxiliaries)
|| hasPassiveProgressiveAuxiliary(kids, passiveAuxiliaries)) {
// String[] how = new String[] {"left", "VP", "ADJP", "NP"};
// Including NP etc seems okay for copular sentences but is
// problematic for other auxiliaries, like 'he has an answer'
// But maybe doing ADJP is fine!
String[] how = {"left", "VP", "ADJP"};
Tree pti = traverseLocate(kids, how, false);
if (DEBUG) {
System.err.println("Determined head (case 1) for " + t.value() + " is: " + pti);
}
if (pti != null) {
return pti;
} else {
// System.err.println("------");
// System.err.println("SemanticHeadFinder failed to reassign head for");
// t.pennPrint(System.err);
// System.err.println("------");
}
}
// looks for copular verbs
if (hasVerbalAuxiliary(kids, copulars) && !isExistential(t, parent) && !isWHQ(t, parent)) {
String[] how;
if (motherCat.equals("SQ")) {
how = new String[] {"right", "VP", "ADJP", "NP", "WHADJP", "WHNP"};
} else {
how = new String[] {"left", "VP", "ADJP", "NP", "WHADJP", "WHNP"};
}
Tree pti = traverseLocate(kids, how, false);
// don't allow a temporal to become head
if (pti != null && pti.label() != null && pti.label().value().contains("-TMP")) {
pti = null;
}
// In SQ, only allow an NP to become head if there is another one to the left (then it's
// probably predicative)
if (motherCat.equals("SQ")
&& pti != null
&& pti.label() != null
&& pti.label().value().startsWith("NP")) {
boolean foundAnotherNp = false;
for (Tree kid : kids) {
if (kid == pti) {
break;
} else if (kid.label() != null && kid.label().value().startsWith("NP")) {
foundAnotherNp = true;
break;
}
}
if (!foundAnotherNp) {
pti = null;
}
}
if (DEBUG) {
System.err.println("Determined head (case 2) for " + t.value() + " is: " + pti);
}
if (pti != null) {
return pti;
} else {
if (DEBUG) {
System.err.println("------");
System.err.println("SemanticHeadFinder failed to reassign head for");
t.pennPrint(System.err);
System.err.println("------");
}
}
}
}
Tree hd = super.determineNonTrivialHead(t, parent);
if (DEBUG) {
System.err.println("Determined head (case 3) for " + t.value() + " is: " + hd);
}
return hd;
}
/**
* for testing -- CURRENTLY BROKEN!!!
*
* @param args input dir and output filename
* @throws IOException
*/
public static void main(String[] args) throws IOException {
if (args.length != 3) {
throw new RuntimeException("args: treebankPath trainNums testNums");
}
ChineseTreebankParserParams ctpp = new ChineseTreebankParserParams();
ctpp.charTags = true;
// TODO: these options are getting clobbered by reading in the
// parser object (unless it's a text file parser?)
Options op = new Options(ctpp);
op.doDep = false;
op.testOptions.maxLength = 90;
LexicalizedParser lp;
try {
FileFilter trainFilt = new NumberRangesFileFilter(args[1], false);
lp = LexicalizedParser.trainFromTreebank(args[0], trainFilt, op);
try {
String filename = "chineseCharTagPCFG.ser.gz";
System.err.println("Writing parser in serialized format to file " + filename + ' ');
System.err.flush();
ObjectOutputStream out = IOUtils.writeStreamFromString(filename);
out.writeObject(lp);
out.close();
System.err.println("done.");
} catch (IOException ioe) {
ioe.printStackTrace();
}
} catch (IllegalArgumentException e) {
lp = LexicalizedParser.loadModel(args[1], op);
}
FileFilter testFilt = new NumberRangesFileFilter(args[2], false);
MemoryTreebank testTreebank = ctpp.memoryTreebank();
testTreebank.loadPath(new File(args[0]), testFilt);
PrintWriter pw =
new PrintWriter(new OutputStreamWriter(new FileOutputStream("out.chi"), "GB18030"), true);
WordCatEquivalenceClasser eqclass = new WordCatEquivalenceClasser();
WordCatEqualityChecker eqcheck = new WordCatEqualityChecker();
EquivalenceClassEval eval = new EquivalenceClassEval(eqclass, eqcheck);
// System.out.println("Preterminals:" + preterminals);
System.out.println("Testing...");
for (Tree gold : testTreebank) {
Tree tree;
try {
tree = lp.parseTree(gold.yieldHasWord());
if (tree == null) {
System.out.println("Failed to parse " + gold.yieldHasWord());
continue;
}
} catch (Exception e) {
e.printStackTrace();
continue;
}
gold = gold.firstChild();
pw.println(Sentence.listToString(gold.preTerminalYield()));
pw.println(Sentence.listToString(gold.yield()));
gold.pennPrint(pw);
pw.println(tree.preTerminalYield());
pw.println(tree.yield());
tree.pennPrint(pw);
// Collection allBrackets = WordCatConstituent.allBrackets(tree);
// Collection goldBrackets = WordCatConstituent.allBrackets(gold);
// eval.eval(allBrackets, goldBrackets);
eval.displayLast();
}
System.out.println();
System.out.println();
eval.display();
}
public static void main(String[] args) {
Options op = new Options(new EnglishTreebankParserParams());
// op.tlpParams may be changed to something else later, so don't use it till
// after options are parsed.
System.out.println("Currently " + new Date());
System.out.print("Invoked with arguments:");
for (String arg : args) {
System.out.print(" " + arg);
}
System.out.println();
String path = "/u/nlp/stuff/corpora/Treebank3/parsed/mrg/wsj";
int trainLow = 200, trainHigh = 2199, testLow = 2200, testHigh = 2219;
String serializeFile = null;
int i = 0;
while (i < args.length && args[i].startsWith("-")) {
if (args[i].equalsIgnoreCase("-path") && (i + 1 < args.length)) {
path = args[i + 1];
i += 2;
} else if (args[i].equalsIgnoreCase("-train") && (i + 2 < args.length)) {
trainLow = Integer.parseInt(args[i + 1]);
trainHigh = Integer.parseInt(args[i + 2]);
i += 3;
} else if (args[i].equalsIgnoreCase("-test") && (i + 2 < args.length)) {
testLow = Integer.parseInt(args[i + 1]);
testHigh = Integer.parseInt(args[i + 2]);
i += 3;
} else if (args[i].equalsIgnoreCase("-serialize") && (i + 1 < args.length)) {
serializeFile = args[i + 1];
i += 2;
} else if (args[i].equalsIgnoreCase("-tLPP") && (i + 1 < args.length)) {
try {
op.tlpParams = (TreebankLangParserParams) Class.forName(args[i + 1]).newInstance();
} catch (ClassNotFoundException e) {
System.err.println("Class not found: " + args[i + 1]);
} catch (InstantiationException e) {
System.err.println("Couldn't instantiate: " + args[i + 1] + ": " + e.toString());
} catch (IllegalAccessException e) {
System.err.println("illegal access" + e);
}
i += 2;
} else if (args[i].equals("-encoding")) {
// sets encoding for TreebankLangParserParams
op.tlpParams.setInputEncoding(args[i + 1]);
op.tlpParams.setOutputEncoding(args[i + 1]);
i += 2;
} else {
i = op.setOptionOrWarn(args, i);
}
}
// System.out.println(tlpParams.getClass());
TreebankLanguagePack tlp = op.tlpParams.treebankLanguagePack();
Train.sisterSplitters = new HashSet(Arrays.asList(op.tlpParams.sisterSplitters()));
// BinarizerFactory.TreeAnnotator.setTreebankLang(tlpParams);
PrintWriter pw = op.tlpParams.pw();
Test.display();
Train.display();
op.display();
op.tlpParams.display();
// setup tree transforms
Treebank trainTreebank = op.tlpParams.memoryTreebank();
MemoryTreebank testTreebank = op.tlpParams.testMemoryTreebank();
// Treebank blippTreebank = ((EnglishTreebankParserParams) tlpParams).diskTreebank();
// String blippPath = "/afs/ir.stanford.edu/data/linguistic-data/BLLIP-WSJ/";
// blippTreebank.loadPath(blippPath, "", true);
Timing.startTime();
System.err.print("Reading trees...");
testTreebank.loadPath(path, new NumberRangeFileFilter(testLow, testHigh, true));
if (Test.increasingLength) {
Collections.sort(testTreebank, new TreeLengthComparator());
}
trainTreebank.loadPath(path, new NumberRangeFileFilter(trainLow, trainHigh, true));
Timing.tick("done.");
System.err.print("Binarizing trees...");
TreeAnnotatorAndBinarizer binarizer = null;
if (!Train.leftToRight) {
binarizer =
new TreeAnnotatorAndBinarizer(op.tlpParams, op.forceCNF, !Train.outsideFactor(), true);
} else {
binarizer =
new TreeAnnotatorAndBinarizer(
op.tlpParams.headFinder(),
new LeftHeadFinder(),
op.tlpParams,
op.forceCNF,
!Train.outsideFactor(),
true);
}
CollinsPuncTransformer collinsPuncTransformer = null;
if (Train.collinsPunc) {
collinsPuncTransformer = new CollinsPuncTransformer(tlp);
}
TreeTransformer debinarizer = new Debinarizer(op.forceCNF);
List<Tree> binaryTrainTrees = new ArrayList<Tree>();
if (Train.selectiveSplit) {
Train.splitters =
ParentAnnotationStats.getSplitCategories(
trainTreebank,
Train.tagSelectiveSplit,
0,
Train.selectiveSplitCutOff,
Train.tagSelectiveSplitCutOff,
op.tlpParams.treebankLanguagePack());
if (Train.deleteSplitters != null) {
List<String> deleted = new ArrayList<String>();
for (String del : Train.deleteSplitters) {
String baseDel = tlp.basicCategory(del);
boolean checkBasic = del.equals(baseDel);
for (Iterator<String> it = Train.splitters.iterator(); it.hasNext(); ) {
String elem = it.next();
String baseElem = tlp.basicCategory(elem);
boolean delStr = checkBasic && baseElem.equals(baseDel) || elem.equals(del);
if (delStr) {
it.remove();
deleted.add(elem);
}
}
}
System.err.println("Removed from vertical splitters: " + deleted);
}
}
if (Train.selectivePostSplit) {
TreeTransformer myTransformer = new TreeAnnotator(op.tlpParams.headFinder(), op.tlpParams);
Treebank annotatedTB = trainTreebank.transform(myTransformer);
Train.postSplitters =
ParentAnnotationStats.getSplitCategories(
annotatedTB,
true,
0,
Train.selectivePostSplitCutOff,
Train.tagSelectivePostSplitCutOff,
op.tlpParams.treebankLanguagePack());
}
if (Train.hSelSplit) {
binarizer.setDoSelectiveSplit(false);
for (Tree tree : trainTreebank) {
if (Train.collinsPunc) {
tree = collinsPuncTransformer.transformTree(tree);
}
// tree.pennPrint(tlpParams.pw());
tree = binarizer.transformTree(tree);
// binaryTrainTrees.add(tree);
}
binarizer.setDoSelectiveSplit(true);
}
for (Tree tree : trainTreebank) {
if (Train.collinsPunc) {
tree = collinsPuncTransformer.transformTree(tree);
}
tree = binarizer.transformTree(tree);
binaryTrainTrees.add(tree);
}
if (Test.verbose) {
binarizer.dumpStats();
}
List<Tree> binaryTestTrees = new ArrayList<Tree>();
for (Tree tree : testTreebank) {
if (Train.collinsPunc) {
tree = collinsPuncTransformer.transformTree(tree);
}
tree = binarizer.transformTree(tree);
binaryTestTrees.add(tree);
}
Timing.tick("done."); // binarization
BinaryGrammar bg = null;
UnaryGrammar ug = null;
DependencyGrammar dg = null;
// DependencyGrammar dgBLIPP = null;
Lexicon lex = null;
// extract grammars
Extractor bgExtractor = new BinaryGrammarExtractor();
// Extractor bgExtractor = new SmoothedBinaryGrammarExtractor();//new BinaryGrammarExtractor();
// Extractor lexExtractor = new LexiconExtractor();
// Extractor dgExtractor = new DependencyMemGrammarExtractor();
Extractor dgExtractor = new MLEDependencyGrammarExtractor(op);
if (op.doPCFG) {
System.err.print("Extracting PCFG...");
Pair bgug = null;
if (Train.cheatPCFG) {
List allTrees = new ArrayList(binaryTrainTrees);
allTrees.addAll(binaryTestTrees);
bgug = (Pair) bgExtractor.extract(allTrees);
} else {
bgug = (Pair) bgExtractor.extract(binaryTrainTrees);
}
bg = (BinaryGrammar) bgug.second;
bg.splitRules();
ug = (UnaryGrammar) bgug.first;
ug.purgeRules();
Timing.tick("done.");
}
System.err.print("Extracting Lexicon...");
lex = op.tlpParams.lex(op.lexOptions);
lex.train(binaryTrainTrees);
Timing.tick("done.");
if (op.doDep) {
System.err.print("Extracting Dependencies...");
binaryTrainTrees.clear();
// dgBLIPP = (DependencyGrammar) dgExtractor.extract(new
// ConcatenationIterator(trainTreebank.iterator(),blippTreebank.iterator()),new
// TransformTreeDependency(tlpParams,true));
DependencyGrammar dg1 =
(DependencyGrammar)
dgExtractor.extract(
trainTreebank.iterator(), new TransformTreeDependency(op.tlpParams, true));
// dgBLIPP=(DependencyGrammar)dgExtractor.extract(blippTreebank.iterator(),new
// TransformTreeDependency(tlpParams));
// dg = (DependencyGrammar) dgExtractor.extract(new
// ConcatenationIterator(trainTreebank.iterator(),blippTreebank.iterator()),new
// TransformTreeDependency(tlpParams));
// dg=new DependencyGrammarCombination(dg1,dgBLIPP,2);
// dg = (DependencyGrammar) dgExtractor.extract(binaryTrainTrees); //uses information whether
// the words are known or not, discards unknown words
Timing.tick("done.");
// System.out.print("Extracting Unknown Word Model...");
// UnknownWordModel uwm = (UnknownWordModel)uwmExtractor.extract(binaryTrainTrees);
// Timing.tick("done.");
System.out.print("Tuning Dependency Model...");
dg.tune(binaryTestTrees);
// System.out.println("TUNE DEPS: "+tuneDeps);
Timing.tick("done.");
}
BinaryGrammar boundBG = bg;
UnaryGrammar boundUG = ug;
GrammarProjection gp = new NullGrammarProjection(bg, ug);
// serialization
if (serializeFile != null) {
System.err.print("Serializing parser...");
LexicalizedParser.saveParserDataToSerialized(
new ParserData(lex, bg, ug, dg, Numberer.getNumberers(), op), serializeFile);
Timing.tick("done.");
}
// test: pcfg-parse and output
ExhaustivePCFGParser parser = null;
if (op.doPCFG) {
parser = new ExhaustivePCFGParser(boundBG, boundUG, lex, op);
}
ExhaustiveDependencyParser dparser =
((op.doDep && !Test.useFastFactored) ? new ExhaustiveDependencyParser(dg, lex, op) : null);
Scorer scorer = (op.doPCFG ? new TwinScorer(new ProjectionScorer(parser, gp), dparser) : null);
// Scorer scorer = parser;
BiLexPCFGParser bparser = null;
if (op.doPCFG && op.doDep) {
bparser =
(Test.useN5)
? new BiLexPCFGParser.N5BiLexPCFGParser(
scorer, parser, dparser, bg, ug, dg, lex, op, gp)
: new BiLexPCFGParser(scorer, parser, dparser, bg, ug, dg, lex, op, gp);
}
LabeledConstituentEval pcfgPE = new LabeledConstituentEval("pcfg PE", true, tlp);
LabeledConstituentEval comboPE = new LabeledConstituentEval("combo PE", true, tlp);
AbstractEval pcfgCB = new LabeledConstituentEval.CBEval("pcfg CB", true, tlp);
AbstractEval pcfgTE = new AbstractEval.TaggingEval("pcfg TE");
AbstractEval comboTE = new AbstractEval.TaggingEval("combo TE");
AbstractEval pcfgTEnoPunct = new AbstractEval.TaggingEval("pcfg nopunct TE");
AbstractEval comboTEnoPunct = new AbstractEval.TaggingEval("combo nopunct TE");
AbstractEval depTE = new AbstractEval.TaggingEval("depnd TE");
AbstractEval depDE =
new AbstractEval.DependencyEval("depnd DE", true, tlp.punctuationWordAcceptFilter());
AbstractEval comboDE =
new AbstractEval.DependencyEval("combo DE", true, tlp.punctuationWordAcceptFilter());
if (Test.evalb) {
EvalB.initEVALBfiles(op.tlpParams);
}
// int[] countByLength = new int[Test.maxLength+1];
// use a reflection ruse, so one can run this without needing the tagger
// edu.stanford.nlp.process.SentenceTagger tagger = (Test.preTag ? new
// edu.stanford.nlp.process.SentenceTagger("/u/nlp/data/tagger.params/wsj0-21.holder") : null);
SentenceProcessor tagger = null;
if (Test.preTag) {
try {
Class[] argsClass = new Class[] {String.class};
Object[] arguments =
new Object[] {"/u/nlp/data/pos-tagger/wsj3t0-18-bidirectional/train-wsj-0-18.holder"};
tagger =
(SentenceProcessor)
Class.forName("edu.stanford.nlp.tagger.maxent.MaxentTagger")
.getConstructor(argsClass)
.newInstance(arguments);
} catch (Exception e) {
System.err.println(e);
System.err.println("Warning: No pretagging of sentences will be done.");
}
}
for (int tNum = 0, ttSize = testTreebank.size(); tNum < ttSize; tNum++) {
Tree tree = testTreebank.get(tNum);
int testTreeLen = tree.yield().size();
if (testTreeLen > Test.maxLength) {
continue;
}
Tree binaryTree = binaryTestTrees.get(tNum);
// countByLength[testTreeLen]++;
System.out.println("-------------------------------------");
System.out.println("Number: " + (tNum + 1));
System.out.println("Length: " + testTreeLen);
// tree.pennPrint(pw);
// System.out.println("XXXX The binary tree is");
// binaryTree.pennPrint(pw);
// System.out.println("Here are the tags in the lexicon:");
// System.out.println(lex.showTags());
// System.out.println("Here's the tagnumberer:");
// System.out.println(Numberer.getGlobalNumberer("tags").toString());
long timeMil1 = System.currentTimeMillis();
Timing.tick("Starting parse.");
if (op.doPCFG) {
// System.err.println(Test.forceTags);
if (Test.forceTags) {
if (tagger != null) {
// System.out.println("Using a tagger to set tags");
// System.out.println("Tagged sentence as: " +
// tagger.processSentence(cutLast(wordify(binaryTree.yield()))).toString(false));
parser.parse(addLast(tagger.processSentence(cutLast(wordify(binaryTree.yield())))));
} else {
// System.out.println("Forcing tags to match input.");
parser.parse(cleanTags(binaryTree.taggedYield(), tlp));
}
} else {
// System.out.println("XXXX Parsing " + binaryTree.yield());
parser.parse(binaryTree.yield());
}
// Timing.tick("Done with pcfg phase.");
}
if (op.doDep) {
dparser.parse(binaryTree.yield());
// Timing.tick("Done with dependency phase.");
}
boolean bothPassed = false;
if (op.doPCFG && op.doDep) {
bothPassed = bparser.parse(binaryTree.yield());
// Timing.tick("Done with combination phase.");
}
long timeMil2 = System.currentTimeMillis();
long elapsed = timeMil2 - timeMil1;
System.err.println("Time: " + ((int) (elapsed / 100)) / 10.00 + " sec.");
// System.out.println("PCFG Best Parse:");
Tree tree2b = null;
Tree tree2 = null;
// System.out.println("Got full best parse...");
if (op.doPCFG) {
tree2b = parser.getBestParse();
tree2 = debinarizer.transformTree(tree2b);
}
// System.out.println("Debinarized parse...");
// tree2.pennPrint();
// System.out.println("DepG Best Parse:");
Tree tree3 = null;
Tree tree3db = null;
if (op.doDep) {
tree3 = dparser.getBestParse();
// was: but wrong Tree tree3db = debinarizer.transformTree(tree2);
tree3db = debinarizer.transformTree(tree3);
tree3.pennPrint(pw);
}
// tree.pennPrint();
// ((Tree)binaryTrainTrees.get(tNum)).pennPrint();
// System.out.println("Combo Best Parse:");
Tree tree4 = null;
if (op.doPCFG && op.doDep) {
try {
tree4 = bparser.getBestParse();
if (tree4 == null) {
tree4 = tree2b;
}
} catch (NullPointerException e) {
System.err.println("Blocked, using PCFG parse!");
tree4 = tree2b;
}
}
if (op.doPCFG && !bothPassed) {
tree4 = tree2b;
}
// tree4.pennPrint();
if (op.doDep) {
depDE.evaluate(tree3, binaryTree, pw);
depTE.evaluate(tree3db, tree, pw);
}
TreeTransformer tc = op.tlpParams.collinizer();
TreeTransformer tcEvalb = op.tlpParams.collinizerEvalb();
Tree tree4b = null;
if (op.doPCFG) {
// System.out.println("XXXX Best PCFG was: ");
// tree2.pennPrint();
// System.out.println("XXXX Transformed best PCFG is: ");
// tc.transformTree(tree2).pennPrint();
// System.out.println("True Best Parse:");
// tree.pennPrint();
// tc.transformTree(tree).pennPrint();
pcfgPE.evaluate(tc.transformTree(tree2), tc.transformTree(tree), pw);
pcfgCB.evaluate(tc.transformTree(tree2), tc.transformTree(tree), pw);
if (op.doDep) {
comboDE.evaluate((bothPassed ? tree4 : tree3), binaryTree, pw);
tree4b = tree4;
tree4 = debinarizer.transformTree(tree4);
if (op.nodePrune) {
NodePruner np = new NodePruner(parser, debinarizer);
tree4 = np.prune(tree4);
}
// tree4.pennPrint();
comboPE.evaluate(tc.transformTree(tree4), tc.transformTree(tree), pw);
}
// pcfgTE.evaluate(tree2, tree);
pcfgTE.evaluate(tcEvalb.transformTree(tree2), tcEvalb.transformTree(tree), pw);
pcfgTEnoPunct.evaluate(tc.transformTree(tree2), tc.transformTree(tree), pw);
if (op.doDep) {
comboTE.evaluate(tcEvalb.transformTree(tree4), tcEvalb.transformTree(tree), pw);
comboTEnoPunct.evaluate(tc.transformTree(tree4), tc.transformTree(tree), pw);
}
System.out.println("PCFG only: " + parser.scoreBinarizedTree(tree2b, 0));
// tc.transformTree(tree2).pennPrint();
tree2.pennPrint(pw);
if (op.doDep) {
System.out.println("Combo: " + parser.scoreBinarizedTree(tree4b, 0));
// tc.transformTree(tree4).pennPrint(pw);
tree4.pennPrint(pw);
}
System.out.println("Correct:" + parser.scoreBinarizedTree(binaryTree, 0));
/*
if (parser.scoreBinarizedTree(tree2b,true) < parser.scoreBinarizedTree(binaryTree,true)) {
System.out.println("SCORE INVERSION");
parser.validateBinarizedTree(binaryTree,0);
}
*/
tree.pennPrint(pw);
} // end if doPCFG
if (Test.evalb) {
if (op.doPCFG && op.doDep) {
EvalB.writeEVALBline(tcEvalb.transformTree(tree), tcEvalb.transformTree(tree4));
} else if (op.doPCFG) {
EvalB.writeEVALBline(tcEvalb.transformTree(tree), tcEvalb.transformTree(tree2));
} else if (op.doDep) {
EvalB.writeEVALBline(tcEvalb.transformTree(tree), tcEvalb.transformTree(tree3db));
}
}
} // end for each tree in test treebank
if (Test.evalb) {
EvalB.closeEVALBfiles();
}
// Test.display();
if (op.doPCFG) {
pcfgPE.display(false, pw);
System.out.println("Grammar size: " + Numberer.getGlobalNumberer("states").total());
pcfgCB.display(false, pw);
if (op.doDep) {
comboPE.display(false, pw);
}
pcfgTE.display(false, pw);
pcfgTEnoPunct.display(false, pw);
if (op.doDep) {
comboTE.display(false, pw);
comboTEnoPunct.display(false, pw);
}
}
if (op.doDep) {
depTE.display(false, pw);
depDE.display(false, pw);
}
if (op.doPCFG && op.doDep) {
comboDE.display(false, pw);
}
// pcfgPE.printGoodBad();
}