public static int getTotalCount(HashMap<String, TokenDetails> map) {
Iterator<Map.Entry<String, TokenDetails>> it = map.entrySet().iterator();
int count = 0;
while (it.hasNext()) {
Map.Entry<String, TokenDetails> pair = (Map.Entry<String, TokenDetails>) it.next();
TokenDetails td = pair.getValue();
count = count + td.getCount();
}
return count;
}
public static ArrayList<TokenDetails> filterProbabilities(ArrayList<TokenDetails> tdList) {
int size = tdList.size();
ArrayList<TokenDetails> tmpList = new ArrayList<TokenDetails>();
for (int i = 0; i < size; i++) {
TokenDetails td = tdList.get(i);
// System.out.println(td.getProb());
if ((td.getProb() >= Utility.probabilityThreshold)
&& (td.getCount() >= Utility.countThreshold)) {
tmpList.add(td);
}
}
return tmpList;
}
/**
* Given a list and a ngram, this method will delete all those ngrams present in the list which
* are superstring of the given ngram and return the updated list.
*
* @param list
* @param ngrams
* @return
*/
public static ArrayList<TokenDetails> deleteNgramsSubsumed(
ArrayList<TokenDetails> list, String ngrams) {
ArrayList<TokenDetails> returnList = new ArrayList<TokenDetails>();
int size = list.size();
for (int i = 0; i < size; i++) {
TokenDetails td = list.get(i);
String s = td.getToken();
if (!Utility.isSubsumedBy(ngrams, s)) {
// System.out.println("Token added:"+s);
returnList.add(td);
} else {
System.out.println("Removing token:" + s);
}
}
return returnList;
}