/** For testing: loads a treebank and prints the trees. */ public static void main(String[] args) { TreebankLangParserParams tlpp = new ChineseTreebankParserParams(); System.out.println("Default encoding is: " + tlpp.diskTreebank().encoding()); if (args.length < 2) { printlnErr( "Usage: edu.stanford.nlp.parser.lexparser.ChineseTreebankParserParams treesPath fileRange"); } else { Treebank m = tlpp.diskTreebank(); m.loadPath(args[0], new NumberRangesFileFilter(args[1], false)); for (Tree t : m) { t.pennPrint(tlpp.pw()); } System.out.println("There were " + m.size() + " trees."); } }
/** * Usage: java edu.stanford.nlp.trees.tregex.tsurgeon.Tsurgeon [-s] -treeFile file-with-trees [-po * matching-pattern operation] operation-file-1 operation-file-2 ... operation-file-n * * <h4>Arguments:</h4> * * Each argument should be the name of a transformation file that contains a list of pattern and * transformation operation list pairs. That is, it is a sequence of pairs of a {@link * TregexPattern} pattern on one or more lines, then a blank line (empty or whitespace), then a * list of transformation operations one per line (as specified by <b>Legal operation syntax</b> * below) to apply when the pattern is matched, and then another blank line (empty or whitespace). * Note the need for blank lines: The code crashes if they are not present as separators (although * the blank line at the end of the file can be omitted). The script file can include comment * lines, either whole comment lines or trailing comments introduced by %, which extend to the end * of line. A needed percent mark can be escaped by a preceding backslash. * * <p>For example, if you want to excise an SBARQ node whenever it is the parent of an SQ node, * and relabel the SQ node to S, your transformation file would look like this: * * <blockquote> * * <code> * SBARQ=n1 < SQ=n2<br> * <br> * excise n1 n1<br> * relabel n2 S * </code> * * </blockquote> * * <p> * * <h4>Options:</h4> * * <ul> * <li><code>-treeFile <filename></code> specify the name of the file that has the trees * you want to transform. * <li><code>-po <matchPattern> <operation></code> Apply a single operation to * every tree using the specified match pattern and the specified operation. Use this option * when you want to quickly try the effect of one pattern/surgery combination, and are too * lazy to write a transformation file. * <li><code>-s</code> Print each output tree on one line (default is pretty-printing). * <li><code>-m</code> For every tree that had a matching pattern, print "before" (prepended as * "Operated on:") and "after" (prepended as "Result:"). Unoperated trees just pass through * the transducer as usual. * <li><code>-encoding X</code> Uses character set X for input and output of trees. * <li><code>-macros <filename></code> A file of macros to use on the tregex pattern. * Macros should be one per line, with original and replacement separated by tabs. * <li><code>-hf <headfinder-class-name></code> use the specified {@link HeadFinder} class * to determine headship relations. * <li><code>-hfArg <string></code> pass a string argument in to the {@link HeadFinder} * class's constructor. <code>-hfArg</code> can be used multiple times to pass in multiple * arguments. * <li><code>-trf <TreeReaderFactory-class-name></code> use the specified {@link * TreeReaderFactory} class to read trees from files. * </ul> * * <h4>Legal operation syntax:</h4> * * <ul> * <li><code>delete <name></code> deletes the node and everything below it. * <li><code>prune <name></code> Like delete, but if, after the pruning, the parent has * no children anymore, the parent is pruned too. Pruning continues to affect all ancestors * until one is found with remaining children. This may result in a null tree. * <li><code>excise <name1> <name2></code> The name1 node should either dominate * or be the same as the name2 node. This excises out everything from name1 to name2. All * the children of name2 go into the parent of name1, where name1 was. * <li><code>relabel <name> <new-label></code> Relabels the node to have the new * label. <br> * There are three possible forms: <br> * <code>relabel nodeX VP</code> - for changing a node label to an alphanumeric string <br> * <code>relabel nodeX /''/</code> - for relabeling a node to something that isn't a valid * identifier without quoting <br> * <code>relabel nodeX /^VB(.*)$/verb\\/$1/</code> - for regular expression based * relabeling. In this case, all matches of the regular expression against the node label * are replaced with the replacement String. This has the semantics of Java/Perl's * replaceAll: you may use capturing groups and put them in replacements with $n. For * example, if the pattern is /foo/bar/ and the node matched is "foo", the replaceAll * semantics result in "barbar". If the pattern is /^foo(.*)$/bar$1/ and node matched is * "foofoo", relabel will result in "barfoo". <br> * When using the regex replacement method, you can also use the sequences ={node} and * %{var} in the replacement string to use captured nodes or variable strings in the * replacement string. For example, if the Tregex pattern was "duck=bar" and the relabel is * /foo/={bar}/, "foofoo" will be replaced with "duckduck". <br> * To concatenate two nodes named in the tregex pattern, for example, you can use the * pattern /^.*$/={foo}={bar}/. Note that the ^.*$ is necessary to make sure the regex * pattern only matches and replaces once on the entire node name. <br> * To get an "=" or a "%" in the replacement, using \ escaping. Also, as in the example you * can escape a slash in the middle of the second and third forms with \\/ and \\\\. <br> * <li><code>insert <name> <position></code> or <code> * insert <tree> <position></code> inserts the named node or tree into the * position specified. * <li><code>move <name> <position></code> moves the named node into the * specified position. * <p>Right now the only ways to specify position are: * <p><code>$+ <name></code> the left sister of the named node<br> * <code>$- <name></code> the right sister of the named node<br> * <code>>i <name></code> the i_th daughter of the named node<br> * <code>>-i <name></code> the i_th daughter, counting from the right, of the * named node. * <li><code>replace <name1> <name2></code> deletes name1 and inserts a copy of * name2 in its place. * <li><code>replace <name> <tree> <tree2>...</code> deletes name and * inserts the new tree(s) in its place. If more than one replacement tree is given, each of * the new subtrees will be added in order where the old tree was. Multiple subtrees at the * root is an illegal operation and will throw an exception. * <li>{@code createSubtree <new-label> <name1> [<name2>]} Create a subtree out of all the nodes * from {@code <name1>} through {@code <name2>} and puts the new subtree where that span * used to be. To limit the operation to just one node, elide {@code <name2>}. * <li><code>adjoin <auxiliary_tree> <name></code> Adjoins the specified auxiliary * tree into the named node. The daughters of the target node will become the daughters of * the foot of the auxiliary tree. * <li><code>adjoinH <auxiliary_tree> <name></code> Similar to adjoin, but * preserves the target node and makes it the root of <tree>. (It is still accessible * as <code>name</code>. The root of the auxiliary tree is ignored.) * <li><code>adjoinF <auxiliary_tree> <name></code> Similar to adjoin, but * preserves the target node and makes it the foot of <tree>. (It is still accessible * as <code>name</code>, and retains its status as parent of its children. The root of the * auxiliary tree is ignored.) * <li> * <dt><code>coindex <name1> <name2> ... <nameM> </code> Puts a (Penn * Treebank style) coindexation suffix of the form "-N" on each of nodes name_1 through * name_m. The value of N will be automatically generated in reference to the existing * coindexations in the tree, so that there is never an accidental clash of indices across * things that are not meant to be coindexed. * </ul> * * <p>In the context of <code>adjoin</code>, <code>adjoinH</code>, and <code>adjoinF</code>, an * auxiliary tree is a tree in Penn Treebank format with <code>@</code> on exactly one of the * leaves denoting the foot of the tree. The operations which use the foot use the labeled node. * For example: <br> * Tsurgeon: <code>adjoin (FOO (BAR@)) foo</code> <br> * Tregex: <code>B=foo</code> <br> * Input: <code>(A (B 1 2))</code> Output: <code>(A (FOO (BAR 1 2)))</code> * * <p>Tsurgeon applies the same operation to the same tree for as long as the given tregex * operation matches. This means that infinite loops are very easy to cause. One common situation * where this comes up is with an insert operation will repeats infinitely many times unless you * add an expression to the tregex that matches against the inserted pattern. For example, this * pattern will infinite loop: * * <blockquote> * * <code> * TregexPattern tregex = TregexPattern.compile("S=node << NP"); <br> * TsurgeonPattern tsurgeon = Tsurgeon.parseOperation("insert (NP foo) >-1 node"); * </code> * * </blockquote> * * This pattern, though, will terminate: * * <blockquote> * * <code> * TregexPattern tregex = TregexPattern.compile("S=node << NP !<< foo"); <br> * TsurgeonPattern tsurgeon = Tsurgeon.parseOperation("insert (NP foo) >-1 node"); * </code> * * </blockquote> * * <p>Tsurgeon has (very) limited support for conditional statements. If a pattern is prefaced * with <code>if exists <name></code>, the rest of the pattern will only execute if the * named node was found in the corresponding TregexMatcher. * * @param args a list of names of files each of which contains a single tregex matching pattern * plus a list, one per line, of transformation operations to apply to the matched pattern. * @throws Exception If an I/O or pattern syntax error */ public static void main(String[] args) throws Exception { String headFinderClassName = null; String headFinderOption = "-hf"; String[] headFinderArgs = null; String headFinderArgOption = "-hfArg"; String encoding = "UTF-8"; String encodingOption = "-encoding"; if (args.length == 0) { System.err.println( "Usage: java edu.stanford.nlp.trees.tregex.tsurgeon.Tsurgeon [-s] -treeFile <file-with-trees> [-po <matching-pattern> <operation>] <operation-file-1> <operation-file-2> ... <operation-file-n>"); System.exit(0); } String treePrintFormats; String singleLineOption = "-s"; String verboseOption = "-v"; String matchedOption = "-m"; // if set, then print original form of trees that are matched & thus operated on String patternOperationOption = "-po"; String treeFileOption = "-treeFile"; String trfOption = "-trf"; String macroOption = "-macros"; String macroFilename = ""; Map<String, Integer> flagMap = Generics.newHashMap(); flagMap.put(patternOperationOption, 2); flagMap.put(treeFileOption, 1); flagMap.put(trfOption, 1); flagMap.put(singleLineOption, 0); flagMap.put(encodingOption, 1); flagMap.put(headFinderOption, 1); flagMap.put(macroOption, 1); Map<String, String[]> argsMap = StringUtils.argsToMap(args, flagMap); args = argsMap.get(null); if (argsMap.containsKey(headFinderOption)) headFinderClassName = argsMap.get(headFinderOption)[0]; if (argsMap.containsKey(headFinderArgOption)) headFinderArgs = argsMap.get(headFinderArgOption); if (argsMap.containsKey(verboseOption)) verbose = true; if (argsMap.containsKey(singleLineOption)) treePrintFormats = "oneline,"; else treePrintFormats = "penn,"; if (argsMap.containsKey(encodingOption)) encoding = argsMap.get(encodingOption)[0]; if (argsMap.containsKey(macroOption)) macroFilename = argsMap.get(macroOption)[0]; TreePrint tp = new TreePrint(treePrintFormats, new PennTreebankLanguagePack()); PrintWriter pwOut = new PrintWriter(new OutputStreamWriter(System.out, encoding), true); TreeReaderFactory trf; if (argsMap.containsKey(trfOption)) { String trfClass = argsMap.get(trfOption)[0]; trf = ReflectionLoading.loadByReflection(trfClass); } else { trf = new TregexPattern.TRegexTreeReaderFactory(); } Treebank trees = new DiskTreebank(trf, encoding); if (argsMap.containsKey(treeFileOption)) { trees.loadPath(argsMap.get(treeFileOption)[0]); } List<Pair<TregexPattern, TsurgeonPattern>> ops = new ArrayList<Pair<TregexPattern, TsurgeonPattern>>(); TregexPatternCompiler compiler; if (headFinderClassName == null) { compiler = new TregexPatternCompiler(); } else { HeadFinder hf; if (headFinderArgs == null) { hf = ReflectionLoading.loadByReflection(headFinderClassName); } else { hf = ReflectionLoading.loadByReflection(headFinderClassName, (Object[]) headFinderArgs); } compiler = new TregexPatternCompiler(hf); } Macros.addAllMacros(compiler, macroFilename, encoding); if (argsMap.containsKey(patternOperationOption)) { TregexPattern matchPattern = compiler.compile(argsMap.get(patternOperationOption)[0]); TsurgeonPattern p = parseOperation(argsMap.get(patternOperationOption)[1]); ops.add(new Pair<TregexPattern, TsurgeonPattern>(matchPattern, p)); } else { for (String arg : args) { List<Pair<TregexPattern, TsurgeonPattern>> pairs = getOperationsFromFile(arg, encoding, compiler); for (Pair<TregexPattern, TsurgeonPattern> pair : pairs) { if (verbose) { System.err.println(pair.second()); } ops.add(pair); } } } for (Tree t : trees) { Tree original = t.deepCopy(); Tree result = processPatternsOnTree(ops, t); if (argsMap.containsKey(matchedOption) && matchedOnTree) { pwOut.println("Operated on: "); displayTree(original, tp, pwOut); pwOut.println("Result: "); } displayTree(result, tp, pwOut); } }
public static void main(String[] args) { Options op = new Options(new EnglishTreebankParserParams()); // op.tlpParams may be changed to something else later, so don't use it till // after options are parsed. System.out.println("Currently " + new Date()); System.out.print("Invoked with arguments:"); for (String arg : args) { System.out.print(" " + arg); } System.out.println(); String path = "/u/nlp/stuff/corpora/Treebank3/parsed/mrg/wsj"; int trainLow = 200, trainHigh = 2199, testLow = 2200, testHigh = 2219; String serializeFile = null; int i = 0; while (i < args.length && args[i].startsWith("-")) { if (args[i].equalsIgnoreCase("-path") && (i + 1 < args.length)) { path = args[i + 1]; i += 2; } else if (args[i].equalsIgnoreCase("-train") && (i + 2 < args.length)) { trainLow = Integer.parseInt(args[i + 1]); trainHigh = Integer.parseInt(args[i + 2]); i += 3; } else if (args[i].equalsIgnoreCase("-test") && (i + 2 < args.length)) { testLow = Integer.parseInt(args[i + 1]); testHigh = Integer.parseInt(args[i + 2]); i += 3; } else if (args[i].equalsIgnoreCase("-serialize") && (i + 1 < args.length)) { serializeFile = args[i + 1]; i += 2; } else if (args[i].equalsIgnoreCase("-tLPP") && (i + 1 < args.length)) { try { op.tlpParams = (TreebankLangParserParams) Class.forName(args[i + 1]).newInstance(); } catch (ClassNotFoundException e) { System.err.println("Class not found: " + args[i + 1]); } catch (InstantiationException e) { System.err.println("Couldn't instantiate: " + args[i + 1] + ": " + e.toString()); } catch (IllegalAccessException e) { System.err.println("illegal access" + e); } i += 2; } else if (args[i].equals("-encoding")) { // sets encoding for TreebankLangParserParams op.tlpParams.setInputEncoding(args[i + 1]); op.tlpParams.setOutputEncoding(args[i + 1]); i += 2; } else { i = op.setOptionOrWarn(args, i); } } // System.out.println(tlpParams.getClass()); TreebankLanguagePack tlp = op.tlpParams.treebankLanguagePack(); Train.sisterSplitters = new HashSet(Arrays.asList(op.tlpParams.sisterSplitters())); // BinarizerFactory.TreeAnnotator.setTreebankLang(tlpParams); PrintWriter pw = op.tlpParams.pw(); Test.display(); Train.display(); op.display(); op.tlpParams.display(); // setup tree transforms Treebank trainTreebank = op.tlpParams.memoryTreebank(); MemoryTreebank testTreebank = op.tlpParams.testMemoryTreebank(); // Treebank blippTreebank = ((EnglishTreebankParserParams) tlpParams).diskTreebank(); // String blippPath = "/afs/ir.stanford.edu/data/linguistic-data/BLLIP-WSJ/"; // blippTreebank.loadPath(blippPath, "", true); Timing.startTime(); System.err.print("Reading trees..."); testTreebank.loadPath(path, new NumberRangeFileFilter(testLow, testHigh, true)); if (Test.increasingLength) { Collections.sort(testTreebank, new TreeLengthComparator()); } trainTreebank.loadPath(path, new NumberRangeFileFilter(trainLow, trainHigh, true)); Timing.tick("done."); System.err.print("Binarizing trees..."); TreeAnnotatorAndBinarizer binarizer = null; if (!Train.leftToRight) { binarizer = new TreeAnnotatorAndBinarizer(op.tlpParams, op.forceCNF, !Train.outsideFactor(), true); } else { binarizer = new TreeAnnotatorAndBinarizer( op.tlpParams.headFinder(), new LeftHeadFinder(), op.tlpParams, op.forceCNF, !Train.outsideFactor(), true); } CollinsPuncTransformer collinsPuncTransformer = null; if (Train.collinsPunc) { collinsPuncTransformer = new CollinsPuncTransformer(tlp); } TreeTransformer debinarizer = new Debinarizer(op.forceCNF); List<Tree> binaryTrainTrees = new ArrayList<Tree>(); if (Train.selectiveSplit) { Train.splitters = ParentAnnotationStats.getSplitCategories( trainTreebank, Train.tagSelectiveSplit, 0, Train.selectiveSplitCutOff, Train.tagSelectiveSplitCutOff, op.tlpParams.treebankLanguagePack()); if (Train.deleteSplitters != null) { List<String> deleted = new ArrayList<String>(); for (String del : Train.deleteSplitters) { String baseDel = tlp.basicCategory(del); boolean checkBasic = del.equals(baseDel); for (Iterator<String> it = Train.splitters.iterator(); it.hasNext(); ) { String elem = it.next(); String baseElem = tlp.basicCategory(elem); boolean delStr = checkBasic && baseElem.equals(baseDel) || elem.equals(del); if (delStr) { it.remove(); deleted.add(elem); } } } System.err.println("Removed from vertical splitters: " + deleted); } } if (Train.selectivePostSplit) { TreeTransformer myTransformer = new TreeAnnotator(op.tlpParams.headFinder(), op.tlpParams); Treebank annotatedTB = trainTreebank.transform(myTransformer); Train.postSplitters = ParentAnnotationStats.getSplitCategories( annotatedTB, true, 0, Train.selectivePostSplitCutOff, Train.tagSelectivePostSplitCutOff, op.tlpParams.treebankLanguagePack()); } if (Train.hSelSplit) { binarizer.setDoSelectiveSplit(false); for (Tree tree : trainTreebank) { if (Train.collinsPunc) { tree = collinsPuncTransformer.transformTree(tree); } // tree.pennPrint(tlpParams.pw()); tree = binarizer.transformTree(tree); // binaryTrainTrees.add(tree); } binarizer.setDoSelectiveSplit(true); } for (Tree tree : trainTreebank) { if (Train.collinsPunc) { tree = collinsPuncTransformer.transformTree(tree); } tree = binarizer.transformTree(tree); binaryTrainTrees.add(tree); } if (Test.verbose) { binarizer.dumpStats(); } List<Tree> binaryTestTrees = new ArrayList<Tree>(); for (Tree tree : testTreebank) { if (Train.collinsPunc) { tree = collinsPuncTransformer.transformTree(tree); } tree = binarizer.transformTree(tree); binaryTestTrees.add(tree); } Timing.tick("done."); // binarization BinaryGrammar bg = null; UnaryGrammar ug = null; DependencyGrammar dg = null; // DependencyGrammar dgBLIPP = null; Lexicon lex = null; // extract grammars Extractor bgExtractor = new BinaryGrammarExtractor(); // Extractor bgExtractor = new SmoothedBinaryGrammarExtractor();//new BinaryGrammarExtractor(); // Extractor lexExtractor = new LexiconExtractor(); // Extractor dgExtractor = new DependencyMemGrammarExtractor(); Extractor dgExtractor = new MLEDependencyGrammarExtractor(op); if (op.doPCFG) { System.err.print("Extracting PCFG..."); Pair bgug = null; if (Train.cheatPCFG) { List allTrees = new ArrayList(binaryTrainTrees); allTrees.addAll(binaryTestTrees); bgug = (Pair) bgExtractor.extract(allTrees); } else { bgug = (Pair) bgExtractor.extract(binaryTrainTrees); } bg = (BinaryGrammar) bgug.second; bg.splitRules(); ug = (UnaryGrammar) bgug.first; ug.purgeRules(); Timing.tick("done."); } System.err.print("Extracting Lexicon..."); lex = op.tlpParams.lex(op.lexOptions); lex.train(binaryTrainTrees); Timing.tick("done."); if (op.doDep) { System.err.print("Extracting Dependencies..."); binaryTrainTrees.clear(); // dgBLIPP = (DependencyGrammar) dgExtractor.extract(new // ConcatenationIterator(trainTreebank.iterator(),blippTreebank.iterator()),new // TransformTreeDependency(tlpParams,true)); DependencyGrammar dg1 = (DependencyGrammar) dgExtractor.extract( trainTreebank.iterator(), new TransformTreeDependency(op.tlpParams, true)); // dgBLIPP=(DependencyGrammar)dgExtractor.extract(blippTreebank.iterator(),new // TransformTreeDependency(tlpParams)); // dg = (DependencyGrammar) dgExtractor.extract(new // ConcatenationIterator(trainTreebank.iterator(),blippTreebank.iterator()),new // TransformTreeDependency(tlpParams)); // dg=new DependencyGrammarCombination(dg1,dgBLIPP,2); // dg = (DependencyGrammar) dgExtractor.extract(binaryTrainTrees); //uses information whether // the words are known or not, discards unknown words Timing.tick("done."); // System.out.print("Extracting Unknown Word Model..."); // UnknownWordModel uwm = (UnknownWordModel)uwmExtractor.extract(binaryTrainTrees); // Timing.tick("done."); System.out.print("Tuning Dependency Model..."); dg.tune(binaryTestTrees); // System.out.println("TUNE DEPS: "+tuneDeps); Timing.tick("done."); } BinaryGrammar boundBG = bg; UnaryGrammar boundUG = ug; GrammarProjection gp = new NullGrammarProjection(bg, ug); // serialization if (serializeFile != null) { System.err.print("Serializing parser..."); LexicalizedParser.saveParserDataToSerialized( new ParserData(lex, bg, ug, dg, Numberer.getNumberers(), op), serializeFile); Timing.tick("done."); } // test: pcfg-parse and output ExhaustivePCFGParser parser = null; if (op.doPCFG) { parser = new ExhaustivePCFGParser(boundBG, boundUG, lex, op); } ExhaustiveDependencyParser dparser = ((op.doDep && !Test.useFastFactored) ? new ExhaustiveDependencyParser(dg, lex, op) : null); Scorer scorer = (op.doPCFG ? new TwinScorer(new ProjectionScorer(parser, gp), dparser) : null); // Scorer scorer = parser; BiLexPCFGParser bparser = null; if (op.doPCFG && op.doDep) { bparser = (Test.useN5) ? new BiLexPCFGParser.N5BiLexPCFGParser( scorer, parser, dparser, bg, ug, dg, lex, op, gp) : new BiLexPCFGParser(scorer, parser, dparser, bg, ug, dg, lex, op, gp); } LabeledConstituentEval pcfgPE = new LabeledConstituentEval("pcfg PE", true, tlp); LabeledConstituentEval comboPE = new LabeledConstituentEval("combo PE", true, tlp); AbstractEval pcfgCB = new LabeledConstituentEval.CBEval("pcfg CB", true, tlp); AbstractEval pcfgTE = new AbstractEval.TaggingEval("pcfg TE"); AbstractEval comboTE = new AbstractEval.TaggingEval("combo TE"); AbstractEval pcfgTEnoPunct = new AbstractEval.TaggingEval("pcfg nopunct TE"); AbstractEval comboTEnoPunct = new AbstractEval.TaggingEval("combo nopunct TE"); AbstractEval depTE = new AbstractEval.TaggingEval("depnd TE"); AbstractEval depDE = new AbstractEval.DependencyEval("depnd DE", true, tlp.punctuationWordAcceptFilter()); AbstractEval comboDE = new AbstractEval.DependencyEval("combo DE", true, tlp.punctuationWordAcceptFilter()); if (Test.evalb) { EvalB.initEVALBfiles(op.tlpParams); } // int[] countByLength = new int[Test.maxLength+1]; // use a reflection ruse, so one can run this without needing the tagger // edu.stanford.nlp.process.SentenceTagger tagger = (Test.preTag ? new // edu.stanford.nlp.process.SentenceTagger("/u/nlp/data/tagger.params/wsj0-21.holder") : null); SentenceProcessor tagger = null; if (Test.preTag) { try { Class[] argsClass = new Class[] {String.class}; Object[] arguments = new Object[] {"/u/nlp/data/pos-tagger/wsj3t0-18-bidirectional/train-wsj-0-18.holder"}; tagger = (SentenceProcessor) Class.forName("edu.stanford.nlp.tagger.maxent.MaxentTagger") .getConstructor(argsClass) .newInstance(arguments); } catch (Exception e) { System.err.println(e); System.err.println("Warning: No pretagging of sentences will be done."); } } for (int tNum = 0, ttSize = testTreebank.size(); tNum < ttSize; tNum++) { Tree tree = testTreebank.get(tNum); int testTreeLen = tree.yield().size(); if (testTreeLen > Test.maxLength) { continue; } Tree binaryTree = binaryTestTrees.get(tNum); // countByLength[testTreeLen]++; System.out.println("-------------------------------------"); System.out.println("Number: " + (tNum + 1)); System.out.println("Length: " + testTreeLen); // tree.pennPrint(pw); // System.out.println("XXXX The binary tree is"); // binaryTree.pennPrint(pw); // System.out.println("Here are the tags in the lexicon:"); // System.out.println(lex.showTags()); // System.out.println("Here's the tagnumberer:"); // System.out.println(Numberer.getGlobalNumberer("tags").toString()); long timeMil1 = System.currentTimeMillis(); Timing.tick("Starting parse."); if (op.doPCFG) { // System.err.println(Test.forceTags); if (Test.forceTags) { if (tagger != null) { // System.out.println("Using a tagger to set tags"); // System.out.println("Tagged sentence as: " + // tagger.processSentence(cutLast(wordify(binaryTree.yield()))).toString(false)); parser.parse(addLast(tagger.processSentence(cutLast(wordify(binaryTree.yield()))))); } else { // System.out.println("Forcing tags to match input."); parser.parse(cleanTags(binaryTree.taggedYield(), tlp)); } } else { // System.out.println("XXXX Parsing " + binaryTree.yield()); parser.parse(binaryTree.yield()); } // Timing.tick("Done with pcfg phase."); } if (op.doDep) { dparser.parse(binaryTree.yield()); // Timing.tick("Done with dependency phase."); } boolean bothPassed = false; if (op.doPCFG && op.doDep) { bothPassed = bparser.parse(binaryTree.yield()); // Timing.tick("Done with combination phase."); } long timeMil2 = System.currentTimeMillis(); long elapsed = timeMil2 - timeMil1; System.err.println("Time: " + ((int) (elapsed / 100)) / 10.00 + " sec."); // System.out.println("PCFG Best Parse:"); Tree tree2b = null; Tree tree2 = null; // System.out.println("Got full best parse..."); if (op.doPCFG) { tree2b = parser.getBestParse(); tree2 = debinarizer.transformTree(tree2b); } // System.out.println("Debinarized parse..."); // tree2.pennPrint(); // System.out.println("DepG Best Parse:"); Tree tree3 = null; Tree tree3db = null; if (op.doDep) { tree3 = dparser.getBestParse(); // was: but wrong Tree tree3db = debinarizer.transformTree(tree2); tree3db = debinarizer.transformTree(tree3); tree3.pennPrint(pw); } // tree.pennPrint(); // ((Tree)binaryTrainTrees.get(tNum)).pennPrint(); // System.out.println("Combo Best Parse:"); Tree tree4 = null; if (op.doPCFG && op.doDep) { try { tree4 = bparser.getBestParse(); if (tree4 == null) { tree4 = tree2b; } } catch (NullPointerException e) { System.err.println("Blocked, using PCFG parse!"); tree4 = tree2b; } } if (op.doPCFG && !bothPassed) { tree4 = tree2b; } // tree4.pennPrint(); if (op.doDep) { depDE.evaluate(tree3, binaryTree, pw); depTE.evaluate(tree3db, tree, pw); } TreeTransformer tc = op.tlpParams.collinizer(); TreeTransformer tcEvalb = op.tlpParams.collinizerEvalb(); Tree tree4b = null; if (op.doPCFG) { // System.out.println("XXXX Best PCFG was: "); // tree2.pennPrint(); // System.out.println("XXXX Transformed best PCFG is: "); // tc.transformTree(tree2).pennPrint(); // System.out.println("True Best Parse:"); // tree.pennPrint(); // tc.transformTree(tree).pennPrint(); pcfgPE.evaluate(tc.transformTree(tree2), tc.transformTree(tree), pw); pcfgCB.evaluate(tc.transformTree(tree2), tc.transformTree(tree), pw); if (op.doDep) { comboDE.evaluate((bothPassed ? tree4 : tree3), binaryTree, pw); tree4b = tree4; tree4 = debinarizer.transformTree(tree4); if (op.nodePrune) { NodePruner np = new NodePruner(parser, debinarizer); tree4 = np.prune(tree4); } // tree4.pennPrint(); comboPE.evaluate(tc.transformTree(tree4), tc.transformTree(tree), pw); } // pcfgTE.evaluate(tree2, tree); pcfgTE.evaluate(tcEvalb.transformTree(tree2), tcEvalb.transformTree(tree), pw); pcfgTEnoPunct.evaluate(tc.transformTree(tree2), tc.transformTree(tree), pw); if (op.doDep) { comboTE.evaluate(tcEvalb.transformTree(tree4), tcEvalb.transformTree(tree), pw); comboTEnoPunct.evaluate(tc.transformTree(tree4), tc.transformTree(tree), pw); } System.out.println("PCFG only: " + parser.scoreBinarizedTree(tree2b, 0)); // tc.transformTree(tree2).pennPrint(); tree2.pennPrint(pw); if (op.doDep) { System.out.println("Combo: " + parser.scoreBinarizedTree(tree4b, 0)); // tc.transformTree(tree4).pennPrint(pw); tree4.pennPrint(pw); } System.out.println("Correct:" + parser.scoreBinarizedTree(binaryTree, 0)); /* if (parser.scoreBinarizedTree(tree2b,true) < parser.scoreBinarizedTree(binaryTree,true)) { System.out.println("SCORE INVERSION"); parser.validateBinarizedTree(binaryTree,0); } */ tree.pennPrint(pw); } // end if doPCFG if (Test.evalb) { if (op.doPCFG && op.doDep) { EvalB.writeEVALBline(tcEvalb.transformTree(tree), tcEvalb.transformTree(tree4)); } else if (op.doPCFG) { EvalB.writeEVALBline(tcEvalb.transformTree(tree), tcEvalb.transformTree(tree2)); } else if (op.doDep) { EvalB.writeEVALBline(tcEvalb.transformTree(tree), tcEvalb.transformTree(tree3db)); } } } // end for each tree in test treebank if (Test.evalb) { EvalB.closeEVALBfiles(); } // Test.display(); if (op.doPCFG) { pcfgPE.display(false, pw); System.out.println("Grammar size: " + Numberer.getGlobalNumberer("states").total()); pcfgCB.display(false, pw); if (op.doDep) { comboPE.display(false, pw); } pcfgTE.display(false, pw); pcfgTEnoPunct.display(false, pw); if (op.doDep) { comboTE.display(false, pw); comboTEnoPunct.display(false, pw); } } if (op.doDep) { depTE.display(false, pw); depDE.display(false, pw); } if (op.doPCFG && op.doDep) { comboDE.display(false, pw); } // pcfgPE.printGoodBad(); }