/**
* Extracts location names from unstructured text using the named entity recognizer (NER) feature
* provided by the Apache OpenNLP Name Finder.
*
* @param plainText Contents of text document
* @return List of location names and positions
*/
public List<LocationOccurrence> extractLocationNames(String plainText) {
if (plainText == null) {
throw new IllegalArgumentException(
"plaintext input to extractLocationNames should not be null");
}
List<LocationOccurrence> nerResults = new ArrayList<LocationOccurrence>();
// The values used in these Spans are string character offsets
Span sentenceSpans[] = sentenceDetector.sentPosDetect(plainText);
// Each sentence gets processed on its own
for (Span sentenceSpan : sentenceSpans) {
// find the start and end position of this sentence in the document
String sentence = plainText.substring(sentenceSpan.getStart(), sentenceSpan.getEnd());
// tokenize the text into the required OpenNLP format
String[] tokens = tokenizer.tokenize(sentence);
// the values used in these Spans are string character offsets of each token from the sentence
// beginning
Span[] tokenPositionsWithinSentence = tokenizer.tokenizePos(sentence);
// find the location names in the tokenized text
// the values used in these Spans are NOT string character offsets, they are indices into the
// 'tokens' array
Span names[] = nameFinder.find(tokens);
// for each name that got found, create our corresponding occurrence
for (Span name : names) {
// find offsets relative to the start of the sentence
int beginningOfFirstWord = tokenPositionsWithinSentence[name.getStart()].getStart();
int endOfLastWord =
tokenPositionsWithinSentence[name.getEnd() - 1]
.getEnd(); // -1 because the high end of a Span is noninclusiv
// to get offsets relative to the document as a whole, just add the offset for the sentence
// itself
int startOffsetInDoc = sentenceSpan.getStart() + beginningOfFirstWord;
int endOffsetInDoc = sentenceSpan.getStart() + endOfLastWord;
// look back into the original input string to figure out what the text is that I got a hit
// on
String nameInDocument = plainText.substring(startOffsetInDoc, endOffsetInDoc);
// add to List of results to return
nerResults.add(new LocationOccurrence(nameInDocument, startOffsetInDoc));
}
}
// this is necessary to maintain consistent results across
// multiple runs on the same data, which is what we want
nameFinder.clearAdaptiveData();
return nerResults;
}
public void createModel(String modelType, String trainFile) throws IOException {
Charset charset = Charset.forName("UTF-8");
System.out.println("File path:" + trainFile);
ObjectStream<String> lineStream =
new PlainTextByLineStream(
new FileInputStream(ModelTypes.TRAIN_FILE_BASE_LOCATION + trainFile + ".train"),
charset);
ObjectStream<NameSample> sampleStream = new NameSampleDataStream(lineStream);
TokenNameFinderModel model;
BufferedOutputStream modelOut = null;
try {
Collections.<String, Object>emptyMap();
/** model = NameFinderME.train("en", modelType, sampleStream, null); */
model =
NameFinderME.train(
"en",
modelType,
sampleStream,
(AdaptiveFeatureGenerator) null,
Collections.<String, Object>emptyMap(),
100,
1);
/**
* model= NameFinderME.train("en", modelType, sampleStream,
* Collections.<String,Object>emptyMap(), 70, 1);
*/
} finally {
sampleStream.close();
}
try {
modelOut =
new BufferedOutputStream(
new FileOutputStream(ModelTypes.BIN_FILE_BASE_LOCATION + trainFile + ".bin"));
model.serialize(modelOut);
} finally {
if (modelOut != null) modelOut.close();
}
}
public void analyze(Document document) {
List<Sentence> sentences = document.getSentences();
for (Sentence sentence : sentences) {
Span[] contractionsSpan;
synchronized (this.contractionFinder) {
contractionsSpan = contractionFinder.find(TextUtils.tokensToString(sentence.getTokens()));
}
List<Token> newTokens = sentence.getTokens();
for (int i = contractionsSpan.length - 1; i >= 0; i--) {
int start = contractionsSpan[i].getStart();
String lexeme = sentence.getTokens().get(start).getLexeme();
String[] contractions = ContractionUtility.expand(lexeme);
Token original = newTokens.remove(start);
if (contractions != null) {
for (int j = contractions.length - 1; j >= 0; j--) {
Token token = new TokenImpl(original.getStart(), original.getEnd(), contractions[j]);
newTokens.add(start, token);
String caze = null;
if (j == 0) caze = "B";
else if (j == contractions.length - 1) caze = "E";
else caze = "I";
token.addContext(Analyzers.CONTRACTION_FINDER, caze);
}
} else {
LOGGER.debug("Missing contraction: " + lexeme);
}
}
sentence.setTokens(newTokens);
}
}