public void addTokens(String text, StopWordList swlist) { Tokeniser tkzr = new Tokeniser(text); tkzr.tokenise(); TokenMap tm = tkzr.getTokenMap(); for (Iterator e = tm.entrySet().iterator(); e.hasNext(); ) { Map.Entry kv = (Map.Entry) e.next(); String t = (String) kv.getKey(); if (swlist.contains(t)) continue; addToken(t); } }
public void addTokens(String text) { Tokeniser tkzr = new Tokeniser(text); tkzr.tokenise(); TokenMap tm = tkzr.getTokenMap(); for (Iterator e = tm.entrySet().iterator(); e.hasNext(); ) { Map.Entry kv = (Map.Entry) e.next(); String t = (String) kv.getKey(); addToken(t); } // putAll(tkzr.getTokenMap()); }