public void addTokens(String text, StopWordList swlist) {
Tokeniser tkzr = new Tokeniser(text);
tkzr.tokenise();
TokenMap tm = tkzr.getTokenMap();
for (Iterator e = tm.entrySet().iterator(); e.hasNext(); ) {
Map.Entry kv = (Map.Entry) e.next();
String t = (String) kv.getKey();
if (swlist.contains(t)) continue;
addToken(t);
}
}
public void addTokens(String text) {
Tokeniser tkzr = new Tokeniser(text);
tkzr.tokenise();
TokenMap tm = tkzr.getTokenMap();
for (Iterator e = tm.entrySet().iterator(); e.hasNext(); ) {
Map.Entry kv = (Map.Entry) e.next();
String t = (String) kv.getKey();
addToken(t);
}
// putAll(tkzr.getTokenMap());
}
private boolean addToken(String type) {
String key = isIgnoreCase() ? Tokeniser.fixType(type.toLowerCase()) : Tokeniser.fixType(type);
if (key.equals("")) return false;
return add(key);
}