public static int getTotalCount(HashMap<String, TokenDetails> map) { Iterator<Map.Entry<String, TokenDetails>> it = map.entrySet().iterator(); int count = 0; while (it.hasNext()) { Map.Entry<String, TokenDetails> pair = (Map.Entry<String, TokenDetails>) it.next(); TokenDetails td = pair.getValue(); count = count + td.getCount(); } return count; }
public static ArrayList<TokenDetails> filterProbabilities(ArrayList<TokenDetails> tdList) { int size = tdList.size(); ArrayList<TokenDetails> tmpList = new ArrayList<TokenDetails>(); for (int i = 0; i < size; i++) { TokenDetails td = tdList.get(i); // System.out.println(td.getProb()); if ((td.getProb() >= Utility.probabilityThreshold) && (td.getCount() >= Utility.countThreshold)) { tmpList.add(td); } } return tmpList; }
/** * Given a list and a ngram, this method will delete all those ngrams present in the list which * are superstring of the given ngram and return the updated list. * * @param list * @param ngrams * @return */ public static ArrayList<TokenDetails> deleteNgramsSubsumed( ArrayList<TokenDetails> list, String ngrams) { ArrayList<TokenDetails> returnList = new ArrayList<TokenDetails>(); int size = list.size(); for (int i = 0; i < size; i++) { TokenDetails td = list.get(i); String s = td.getToken(); if (!Utility.isSubsumedBy(ngrams, s)) { // System.out.println("Token added:"+s); returnList.add(td); } else { System.out.println("Removing token:" + s); } } return returnList; }