示例#1
0
 public static int getTotalCount(HashMap<String, TokenDetails> map) {
   Iterator<Map.Entry<String, TokenDetails>> it = map.entrySet().iterator();
   int count = 0;
   while (it.hasNext()) {
     Map.Entry<String, TokenDetails> pair = (Map.Entry<String, TokenDetails>) it.next();
     TokenDetails td = pair.getValue();
     count = count + td.getCount();
   }
   return count;
 }
示例#2
0
 public static ArrayList<TokenDetails> filterProbabilities(ArrayList<TokenDetails> tdList) {
   int size = tdList.size();
   ArrayList<TokenDetails> tmpList = new ArrayList<TokenDetails>();
   for (int i = 0; i < size; i++) {
     TokenDetails td = tdList.get(i);
     // System.out.println(td.getProb());
     if ((td.getProb() >= Utility.probabilityThreshold)
         && (td.getCount() >= Utility.countThreshold)) {
       tmpList.add(td);
     }
   }
   return tmpList;
 }
示例#3
0
 /**
  * Given a list and a ngram, this method will delete all those ngrams present in the list which
  * are superstring of the given ngram and return the updated list.
  *
  * @param list
  * @param ngrams
  * @return
  */
 public static ArrayList<TokenDetails> deleteNgramsSubsumed(
     ArrayList<TokenDetails> list, String ngrams) {
   ArrayList<TokenDetails> returnList = new ArrayList<TokenDetails>();
   int size = list.size();
   for (int i = 0; i < size; i++) {
     TokenDetails td = list.get(i);
     String s = td.getToken();
     if (!Utility.isSubsumedBy(ngrams, s)) {
       // System.out.println("Token added:"+s);
       returnList.add(td);
     } else {
       System.out.println("Removing token:" + s);
     }
   }
   return returnList;
 }