/** splits document 'doc' into sentences, adding 'sentence' annotations */
static void addSentences(Document doc) {
SpecialZoner.findSpecialZones(doc);
Vector<Annotation> textSegments = doc.annotationsOfType("TEXT");
if (textSegments == null) {
System.out.println("No <TEXT> in document");
return;
}
for (Annotation ann : textSegments) {
Span textSpan = ann.span();
// check document case
Ace.monocase = Ace.allLowerCase(doc);
// split into sentences
SentenceSplitter.split(doc, textSpan);
}
Vector<Annotation> sentences = doc.annotationsOfType("sentence");
if (sentences != null) {
int sentNo = 0;
for (Annotation sentence : sentences) {
sentNo++;
sentence.put("ID", "SENT-" + sentNo);
}
}
doc.removeAnnotationsOfType("dateline");
doc.removeAnnotationsOfType("textBreak");
doc.shrink("sentence");
}
static void writeDoc1(Document doc, PrintStream out) throws IOException {
Vector<Annotation> entities = doc.annotationsOfType("entity");
if (entities == null) {
System.err.println("No Entity: " + doc);
return;
}
Iterator<Annotation> entityIt = entities.iterator();
int i = 0;
while (entityIt.hasNext()) {
Annotation entity = entityIt.next();
Vector mentions = (Vector) entity.get("mentions");
Iterator mentionIt = mentions.iterator();
String nameType = (String) entity.get("nameType");
while (mentionIt.hasNext()) {
Annotation mention1 = (Annotation) mentionIt.next();
Annotation mention2 = new Annotation("refobj", mention1.span(), new FeatureSet());
mention2.put("objid", Integer.toString(i));
if (nameType != null) {
mention2.put("netype", nameType);
}
doc.addAnnotation(mention2);
}
i++;
}
// remove other annotations.
String[] annotypes = doc.getAnnotationTypes();
for (i = 0; i < annotypes.length; i++) {
String t = annotypes[i];
if (!(t.equals("tagger") || t.equals("refobj") || t.equals("ENAMEX"))) {
doc.removeAnnotationsOfType(t);
}
}
writeDocRaw(doc, out);
return;
}
/**
* train the tagger using the DocumentCollection in file 'trainingCollection'.
* 'trainingCollection' should consist of documents which have been explicitly tagged with
* part-of-speech information.
*/
void train(String trainingCollection) {
for (int i = 0; i < posTable.length; i++)
tagTable[i] = new String[] {"constit", "cat", posTable[i], posTable[i]};
// build ergodic HMM with one state for each POS (plus start and end states)
HMMstate startState = new HMMstate("start", "", WordFeatureHMMemitter.class);
posh.addState(startState);
for (int j = 0; j < posTable.length; j++) startState.addArc(new HMMarc(posTable[j], 0));
HMMstate endState = new HMMstate("end", "", WordFeatureHMMemitter.class);
posh.addState(endState);
for (int i = 0; i < posTable.length; i++) {
String pos = posTable[i];
HMMstate state = new HMMstate(pos, pos, WordFeatureHMMemitter.class);
posh.addState(state);
for (int j = 0; j < posTable.length; j++) state.addArc(new HMMarc(posTable[j], 0));
state.addArc(new HMMarc("end", 0));
}
posh.resolveNames();
posh.resetForTraining();
annotator = new HMMannotator(posh);
annotator.setTagTable(tagTable);
annotator.setBItag(false);
DocumentCollection col = new DocumentCollection(trainingCollection);
col.open();
for (int i = 0; i < col.size(); i++) {
ExternalDocument doc = col.get(i);
doc.open();
System.out.println("Training from " + doc.fileName());
// divide at endmarks (constit cat="."), adding "S" marks
int posn = 0;
int start = posn;
Vector anns;
while ((anns = doc.annotationsAt(posn, "constit")) != null) {
Annotation ann = (Annotation) anns.get(0);
posn = ann.span().end();
String pos = (String) ann.get("cat");
if (pos.equals(".")) {
doc.annotate("S", new Span(start, posn), new FeatureSet());
start = posn;
}
}
annotator.train(doc);
// free up space taken by annotations on document
doc.clearAnnotations();
}
posh.computeProbabilities();
}