/**
   * Builds Jet.Tipster.Document object from Penn treebank corpus.
   *
   * @param in
   * @return
   * @throws IOException
   * @throws InvalidFormatException
   */
  public Treebank load(Reader in) throws IOException, InvalidFormatException {

    List<ParseTreeNode> trees = new ArrayList<ParseTreeNode>();
    PushbackReader input = new PushbackReader(in);

    int start = 0;
    while (true) {
      skipWhitespace(input);
      if (lookAhead(input) == -1) {
        break;
      }

      ParseTreeNode tree = readNode(input);
      trees.add(tree);
      determineSpans(tree, start);
      setAnnotations(tree, null);
      start = tree.end;
    }

    String text = buildDocumentString(trees);
    Document doc = new Document(text);
    for (ParseTreeNode tree : trees) {
      doc.annotate("sentence", new Span(tree.start, tree.end), new FeatureSet());
      annotate(doc, tree);
    }

    return new Treebank(doc, trees);
  }
 /**
  * Adds <B>constit</B> annotations to an existing Document <CODE>doc</CODE> to represent the parse
  * tree structure of a set of trees <CODE>trees</CODE>. This version is provided for parse tree
  * files which include sentence offsets.
  *
  * @param trees list of parse trees
  * @param offsets list of the starting position (in doc) of the text corresponding to each parse
  *     tree
  * @param doc document to which annotations should be added
  * @param targetAnnotation name of annotation to get 'parse' feature pointing to parse tree
  * @param span target span.
  * @param jetCategories if false, use lexical categories from Penn Tree Bank; if true, use
  *     categories from Jet
  */
 public void addAnnotations(
     List<ParseTreeNode> trees,
     List<Integer> offsets,
     Document doc,
     String targetAnnotation,
     Span span,
     boolean jetCategories) {
   if (trees.size() != offsets.size()) {
     System.err.println(
         "PTBReader.addAnnotations:  mismatch between number of "
             + "trees ("
             + trees.size()
             + ") and number of offsets ("
             + offsets.size()
             + ")");
     return;
   }
   for (int i = 0; i < trees.size(); i++) {
     ParseTreeNode tree = trees.get(i);
     int start = offsets.get(i);
     if (start < 0) {
       System.err.println("PTBReader.addAnnotations:  offset missing for " + " parse tree " + i);
       continue;
     }
     int end = (i + 1 == offsets.size()) ? span.end() : offsets.get(i + 1);
     Span sentenceSpan = new Span(start, end);
     addAnnotations(tree, doc, sentenceSpan, jetCategories);
     Vector<Annotation> anns = doc.annotationsAt(start, targetAnnotation);
     if (anns != null && anns.size() > 0) {
       Annotation ann = anns.get(0);
       ann.put("parse", tree.ann);
     }
   }
 }
  /**
   * Adds <B>constit</B> annotations to an existing Document <CODE>doc</CODE> to represent the parse
   * tree structure of a set of trees <CODE>trees</CODE>.
   *
   * @param trees list of parse trees
   * @param doc document to which annotations should be added
   * @param targetAnnotation name of annotation to determine spans to add parse tree annotations.
   * @param span target span.
   * @param jetCategories if false, use lexical categories from Penn Tree Bank; if true, use
   *     categories from Jet
   */
  public void addAnnotations(
      List<ParseTreeNode> trees,
      Document doc,
      String targetAnnotation,
      Span span,
      boolean jetCategories) {
    List<Annotation> targetList = (List<Annotation>) doc.annotationsOfType(targetAnnotation, span);
    Comparator<Annotation> cmp =
        new Comparator<Annotation>() {
          public int compare(Annotation a, Annotation b) {
            return a.span().compareTo(b.span());
          }
        };

    Collections.sort(targetList, cmp);
    if (trees.size() != targetList.size()) {
      System.err.println(
          "PTBReader.addAnnotations:  mismatch between number of "
              + targetAnnotation
              + " ("
              + targetList.size()
              + ") and number of trees ("
              + trees.size()
              + ")");
    }
    int n = Math.min(trees.size(), targetList.size());
    for (int i = 0; i < n; i++) {
      ParseTreeNode tree = trees.get(i);
      addAnnotations(tree, doc, targetList.get(i).span(), jetCategories);
      targetList.get(i).put("parse", tree.ann);
    }
  }
  /**
   * Adds <B>constit</B> annotations to an existing Document <CODE>doc</CODE> to represent the parse
   * tree structure <CODE>tree</CODE>.
   *
   * @param tree the parse tree (for a portion of Document doc)
   * @param doc the document
   * @param span the portion of doc covered by the parse tree
   * @param jetCategories if true, use Jet categories as terminal categories (if false, use
   *     categories read from parse trees)
   */
  public void addAnnotations(ParseTreeNode tree, Document doc, Span span, boolean jetCategories) {
    List<ParseTreeNode> terminalNodes = getTerminalNodes(tree);
    String text = doc.text();
    int offset = span.start();

    for (ParseTreeNode terminal : terminalNodes) {
      while (offset < span.end() && Character.isWhitespace(text.charAt(offset))) {
        offset++;
      }
      for (String skipString : skip) {
        if (text.startsWith(skipString, offset)) {
          offset += skipString.length();
          while (offset < span.end() && Character.isWhitespace(text.charAt(offset))) {
            offset++;
          }
          break;
        }
      }
      // match next terminal node against next word in text
      int matchLength = matchTextToTree(text, offset, terminal.word);
      if (matchLength > 0) {
        int endOffset = offset + matchLength;
        while (endOffset < span.end() && Character.isWhitespace(text.charAt(endOffset))) {
          endOffset++;
        }
        terminal.start = offset;
        terminal.end = endOffset;
        offset = endOffset;
      } else {
        System.err.println(
            "PTBReader.addAnnotations:  "
                + "Cannot determine parse tree offset for word "
                + terminal.word);
        System.err.println("  at document offset " + offset + " in sentence");
        System.err.println("  " + doc.text(span));
        return;
      }
    }

    if (jetCategories) {
      setJetAnnotations(tree, span, doc);
      StatParser.deleteUnusedConstits(doc, span, tree.ann); // <<<
    } else {
      determineNonTerminalSpans(tree, span.start());
      setAnnotations(tree, doc);
    }
  }
  private void annotate(Document doc, ParseTreeNode node) {
    doc.addAnnotation(node.ann);
    if (node.children != null) {
      Annotation[] children = new Annotation[node.children.length];
      for (int i = 0; i < node.children.length; i++) {
        children[i] = node.children[i].ann;
      }
      node.ann.put("children", children);

      for (ParseTreeNode child : node.children) {
        annotate(doc, child);
      }
    }

    if (node.children == null && isAddingTokens) {
      // TODO: adds `case' property
      doc.annotate("token", node.ann.span(), new FeatureSet());
    }
  }
  /**
   * converts a set of Penn TreeBank files into text documents. Invoked by: PTBReader inputDir
   * outputDir. Converts all files with extension .mrg in inputDir to text documents, and writes
   * them into outputDir.
   */
  public static void main(String[] args) throws Exception {
    if (args.length != 2) {
      System.out.println("usage: java " + PTBReader.class.getName() + " ");
      System.exit(1);
    }

    File inputDir = new File(args[0]);
    File outputDir = new File(args[1]);
    PTBReader parser = new PTBReader();
    for (File file : getFiles(new File(args[0]), ".mrg")) {
      String outFilename = removeSuffix(getRelativePath(inputDir, file));
      File outFile = new File(outputDir, outFilename);
      outFile.getParentFile().mkdirs();

      Writer out = new FileWriter(outFile);
      Document doc = parser.load(file).getDocument();
      out.write(doc.text());
      out.close();
    }
  }
 /**
  * Creates annotations for each node in parse tree <CODE>node</NODE>.
  * These annotations are added to the parse tree and to the document
  * <CODE>doc</CODE>.  In constrast to <CODE>setAnnotations</CODE>,
  * the categories used for terminal nodes are Jet categories obtained by
  * Jet tokenization and lexical look-up.  This means that hyphenated
  * items are split, and multi-word names are reduced to a single node.
  *
  * @param node      the root of the parse tree
  * @param treeSpan  the span of the document matching the parse tree
  * @param doc       the document to which annotations will be added
  */
 private void setJetAnnotations(ParseTreeNode node, Span treeSpan, Document doc) {
   StatParser.buildParserInput(doc, treeSpan.start(), treeSpan.end(), false);
   StatParser.fixHyphenatedItems(doc);
   int nameConstitEnd = -1;
   List<ParseTreeNode> terminals = getTerminalNodes(node);
   for (ParseTreeNode terminal : terminals) {
     int terminalEnd = terminal.end;
     // is there a 'name' constituent or 'hyphword' constituent here?
     Vector<Annotation> constits = doc.annotationsAt(terminal.start, "constit");
     Annotation constit = null;
     Annotation nameConstit = null;
     Annotation hyphword = null;
     if (constits != null) {
       for (Annotation c : constits) {
         if (c.get("cat") == "name") {
           nameConstit = c;
         } else if (c.get("cat") == "hyphword") {
           hyphword = c;
         }
         if (constit == null) constit = c;
       }
     }
     if (hyphword != null) {
       nameConstit = null;
       constit = hyphword;
     }
     // if there is a name which is not part of a hyphword, associate the
     // name with this (first) terminal node, and mark any remaining terminal
     // nodes which match tokens in the name as empty
     if (nameConstit != null) {
       terminal.end = nameConstit.end();
       terminal.ann = nameConstit;
       nameConstitEnd = nameConstit.end();
     } else if (nameConstitEnd >= 0) {
       terminal.word = null;
     } else {
       Span span = new Span(terminal.start, terminal.end);
       String pennPOS = ((String) terminal.category).toUpperCase().intern();
       String word = terminal.word;
       terminal.ann = StatParser.buildWordDefn(doc, word, span, constit, pennPOS);
     }
     if (nameConstitEnd == terminalEnd) nameConstitEnd = -1;
   }
   // prune parse tree:  remove a node if it has no word or children
   pruneTree(node);
   determineNonTerminalSpans(node, treeSpan.start());
   // add head links
   if (hr == null) hr = HeadRule.createDefaultRule();
   hr.apply(node);
   // add annotations for non-terminals:
   Jet.Parser.ParseTreeNode.makeParseAnnotations(doc, node);
 }
  public void apply(Document doc, List<Object> values, Span span, DateTime ref) {
    Map params = getParameters();
    String value = (String) params.get("value");
    String diff = (String) params.get("diff");
    String dir = (String) params.get("dir");
    DateTime val = ref;
    if (value != null) {
      value = assignValues(value, values);
      val = new DateTime(value);
    } else if (diff != null) {
      diff = assignValues(diff, values);
      Period period = new Period(diff);

      if (dir == null || dir.equals("plus")) {
        val = ref.plus(period);
      } else if (dir.equals("minus")) {
        val = ref.minus(period);
      }
    } else {
      val = ref;
      // use set_xxx
      for (Map.Entry entry : (Set<Map.Entry>) params.entrySet()) {
        Matcher m = Pattern.compile("set_(.*)").matcher((String) entry.getKey());
        if (m.matches()) {
          String field = assignValues((String) entry.getValue(), values);
          String fieldName = m.group(1);

          if (fieldName.equals("month")) {
            int month = Integer.parseInt(field);
            val = getTimeAnnotator().normalizeMonth(val, month);
          } else if (fieldName.equals("day")) {
            int day = Integer.parseInt(field);
            val = val.withField(DateTimeFieldType.dayOfMonth(), day);
          } else {
            throw new InternalError();
          }
        }
      }
    }

    String formattedDate = formatter.print(val);
    FeatureSet attrs = new FeatureSet();
    attrs.put("VAL", formattedDate);

    doc.annotate("TIMEX2", span, attrs);
  }
  /**
   * Creates annotations for each node in parse tree <CODE>node</NODE>.
   * These annotations are added to the parse tree;  in addition, if
   * Document <CODE>doc</CODE> is non-empty, they are added to the document.
   * <P>
   * Note that this method does not set the "children" attribute.
   *
   * @param node
   * @param doc
   */
  private void setAnnotations(ParseTreeNode node, Document doc) {
    Span span = new Span(node.start, node.end);
    FeatureSet attrs = new FeatureSet();
    attrs.put("cat", node.category);
    if (node.head != 0) {
      attrs.put("head", node.head);
    }
    if (node.function != null) {
      attrs.put("func", node.function);
    }

    node.ann = new Annotation("constit", span, attrs);
    if (doc != null) {
      doc.addAnnotation(node.ann);
    }

    if (node.children != null) {
      for (ParseTreeNode child : node.children) {
        setAnnotations(child, doc);
      }
    }
  }