public NGramFrequencyMap extractFreqs(Sentence annotations) {
String text;
if (removeStopWords) {
Sentence noStopwordsText = removeStopwords(annotations);
text = noStopwordsText.getConcatTokens();
} else text = annotations.toString();
return extractFreqs(text);
}
public NGramFrequencyMap extractFreqs(Text annotations) {
String text;
if (removeStopWords) {
List<Sentence> noStopwordsText = removeStopwords(annotations);
text = "";
for (Sentence sent : noStopwordsText) {
text = text + " " + sent.getConcatTokens();
}
text = text.trim();
} else text = annotations.toString();
return extractFreqs(text);
}
/**
* Assign scores for similarity in n-gram frequency to the given question to every sentence in the
* possible answer.
*/
public void annotate(PossibleAnswer possibleAnswer) {
// count n-gram frequencies of question text
Question question = possibleAnswer.getQuestion();
Text text = possibleAnswer.getAnnotatedText();
NGramFrequencyMap questionFreq;
for (NGramFreqCounter freqCounter : freqCounters) {
questionFreq = freqCounter.extractFreqs(question.getAnnotatedText());
NGramFrequencyMap answerFreq;
Double distance;
for (Sentence sentence : text.getSentences()) {
// count n-gram frequencies for every sentence in the answer-text
answerFreq = freqCounter.extractFreqs(sentence);
// compare to question frequencies
distance = distEval.evaluateDistance(questionFreq, answerFreq);
// store distance in sentence
sentence.addNgramDistance(extractDescription(freqCounter), distance);
}
}
}