/**
* Sets the token list for this utterance. Note that this could be optimized by turning the token
* list directly into the token relation.
*
* @param tokenList the tokenList
*/
private void setTokenList(List tokenList) {
setInputText(tokenList);
Relation relation = createRelation(Relation.TOKEN);
for (Iterator i = tokenList.iterator(); i.hasNext(); ) {
Token token = (Token) i.next();
String tokenWord = token.getWord();
if (tokenWord != null && tokenWord.length() > 0) {
Item item = relation.appendItem();
FeatureSet featureSet = item.getFeatures();
featureSet.setString("name", tokenWord);
featureSet.setString("whitespace", token.getWhitespace());
featureSet.setString("prepunctuation", token.getPrepunctuation());
featureSet.setString("punc", token.getPostpunctuation());
featureSet.setString("file_pos", String.valueOf(token.getPosition()));
featureSet.setString("line_number", String.valueOf(token.getLineNumber()));
}
}
}
