Exemplo n.º 1
0
 /** splits document 'doc' into sentences, adding 'sentence' annotations */
 static void addSentences(Document doc) {
   SpecialZoner.findSpecialZones(doc);
   Vector<Annotation> textSegments = doc.annotationsOfType("TEXT");
   if (textSegments == null) {
     System.out.println("No <TEXT> in document");
     return;
   }
   for (Annotation ann : textSegments) {
     Span textSpan = ann.span();
     // check document case
     Ace.monocase = Ace.allLowerCase(doc);
     // split into sentences
     SentenceSplitter.split(doc, textSpan);
   }
   Vector<Annotation> sentences = doc.annotationsOfType("sentence");
   if (sentences != null) {
     int sentNo = 0;
     for (Annotation sentence : sentences) {
       sentNo++;
       sentence.put("ID", "SENT-" + sentNo);
     }
   }
   doc.removeAnnotationsOfType("dateline");
   doc.removeAnnotationsOfType("textBreak");
   doc.shrink("sentence");
 }
Exemplo n.º 2
0
 static void writeDoc1(Document doc, PrintStream out) throws IOException {
   Vector<Annotation> entities = doc.annotationsOfType("entity");
   if (entities == null) {
     System.err.println("No Entity: " + doc);
     return;
   }
   Iterator<Annotation> entityIt = entities.iterator();
   int i = 0;
   while (entityIt.hasNext()) {
     Annotation entity = entityIt.next();
     Vector mentions = (Vector) entity.get("mentions");
     Iterator mentionIt = mentions.iterator();
     String nameType = (String) entity.get("nameType");
     while (mentionIt.hasNext()) {
       Annotation mention1 = (Annotation) mentionIt.next();
       Annotation mention2 = new Annotation("refobj", mention1.span(), new FeatureSet());
       mention2.put("objid", Integer.toString(i));
       if (nameType != null) {
         mention2.put("netype", nameType);
       }
       doc.addAnnotation(mention2);
     }
     i++;
   }
   // remove other annotations.
   String[] annotypes = doc.getAnnotationTypes();
   for (i = 0; i < annotypes.length; i++) {
     String t = annotypes[i];
     if (!(t.equals("tagger") || t.equals("refobj") || t.equals("ENAMEX"))) {
       doc.removeAnnotationsOfType(t);
     }
   }
   writeDocRaw(doc, out);
   return;
 }
Exemplo n.º 3
0
  /**
   * train the tagger using the DocumentCollection in file 'trainingCollection'.
   * 'trainingCollection' should consist of documents which have been explicitly tagged with
   * part-of-speech information.
   */
  void train(String trainingCollection) {

    for (int i = 0; i < posTable.length; i++)
      tagTable[i] = new String[] {"constit", "cat", posTable[i], posTable[i]};

    // build ergodic HMM with one state for each POS (plus start and end states)

    HMMstate startState = new HMMstate("start", "", WordFeatureHMMemitter.class);
    posh.addState(startState);
    for (int j = 0; j < posTable.length; j++) startState.addArc(new HMMarc(posTable[j], 0));
    HMMstate endState = new HMMstate("end", "", WordFeatureHMMemitter.class);
    posh.addState(endState);
    for (int i = 0; i < posTable.length; i++) {
      String pos = posTable[i];
      HMMstate state = new HMMstate(pos, pos, WordFeatureHMMemitter.class);
      posh.addState(state);
      for (int j = 0; j < posTable.length; j++) state.addArc(new HMMarc(posTable[j], 0));
      state.addArc(new HMMarc("end", 0));
    }
    posh.resolveNames();

    posh.resetForTraining();
    annotator = new HMMannotator(posh);
    annotator.setTagTable(tagTable);
    annotator.setBItag(false);

    DocumentCollection col = new DocumentCollection(trainingCollection);
    col.open();
    for (int i = 0; i < col.size(); i++) {
      ExternalDocument doc = col.get(i);
      doc.open();
      System.out.println("Training from " + doc.fileName());

      // divide at endmarks (constit cat="."), adding "S" marks

      int posn = 0;
      int start = posn;
      Vector anns;
      while ((anns = doc.annotationsAt(posn, "constit")) != null) {
        Annotation ann = (Annotation) anns.get(0);
        posn = ann.span().end();
        String pos = (String) ann.get("cat");
        if (pos.equals(".")) {
          doc.annotate("S", new Span(start, posn), new FeatureSet());
          start = posn;
        }
      }
      annotator.train(doc);
      //  free up space taken by annotations on document
      doc.clearAnnotations();
    }
    posh.computeProbabilities();
  }