/**
* Returns a list of featured thresholded by minPrecision and sorted by their frequency of
* occurrence. precision in this case, is defined as the frequency of majority label over total
* frequency for that feature.
*
* @return list of high precision features.
*/
private List<F> getHighPrecisionFeatures(
GeneralDataset<L, F> dataset, double minPrecision, int maxNumFeatures) {
int[][] feature2label = new int[dataset.numFeatures()][dataset.numClasses()];
for (int f = 0; f < dataset.numFeatures(); f++) Arrays.fill(feature2label[f], 0);
int[][] data = dataset.data;
int[] labels = dataset.labels;
for (int d = 0; d < data.length; d++) {
int label = labels[d];
// System.out.println("datum id:"+d+" label id: "+label);
if (data[d] != null) {
// System.out.println(" number of features:"+data[d].length);
for (int n = 0; n < data[d].length; n++) {
feature2label[data[d][n]][label]++;
}
}
}
Counter<F> feature2freq = new ClassicCounter<F>();
for (int f = 0; f < dataset.numFeatures(); f++) {
int maxF = ArrayMath.max(feature2label[f]);
int total = ArrayMath.sum(feature2label[f]);
double precision = ((double) maxF) / total;
F feature = dataset.featureIndex.get(f);
if (precision >= minPrecision) {
feature2freq.incrementCount(feature, total);
}
}
if (feature2freq.size() > maxNumFeatures) {
Counters.retainTop(feature2freq, maxNumFeatures);
}
// for(F feature : feature2freq.keySet())
// System.out.println(feature+" "+feature2freq.getCount(feature));
// System.exit(0);
return Counters.toSortedList(feature2freq);
}
public static List<String> generateDict(List<String> str, int cutOff) {
Counter<String> freq = new IntCounter<>();
for (String aStr : str) freq.incrementCount(aStr);
List<String> keys = Counters.toSortedList(freq, false);
List<String> dict = new ArrayList<>();
for (String word : keys) {
if (freq.getCount(word) >= cutOff) dict.add(word);
}
return dict;
}
public List<String> selectKeys(ActiveLearningSelectionCriterion criterion) {
Counter<String> weights = uncertainty(criterion);
return Counters.toSortedList(weights);
}
