private void computeEmpiricalStatistics(List<F> geFeatures) {
// allocate memory to the containers and initialize them
geFeature2EmpiricalDist = new double[geFeatures.size()][labeledDataset.labelIndex.size()];
geFeature2DatumList = new ArrayList<List<Integer>>(geFeatures.size());
Map<F, Integer> geFeatureMap = Generics.newHashMap();
Set<Integer> activeUnlabeledExamples = Generics.newHashSet();
for (int n = 0; n < geFeatures.size(); n++) {
F geFeature = geFeatures.get(n);
geFeature2DatumList.add(new ArrayList<Integer>());
Arrays.fill(geFeature2EmpiricalDist[n], 0);
geFeatureMap.put(geFeature, n);
}
// compute the empirical label distribution for each GE feature
for (int i = 0; i < labeledDataset.size(); i++) {
Datum<L, F> datum = labeledDataset.getDatum(i);
int labelID = labeledDataset.labelIndex.indexOf(datum.label());
for (F feature : datum.asFeatures()) {
if (geFeatureMap.containsKey(feature)) {
int geFnum = geFeatureMap.get(feature);
geFeature2EmpiricalDist[geFnum][labelID]++;
}
}
}
// now normalize and smooth the label distribution for each feature.
for (int n = 0; n < geFeatures.size(); n++) {
ArrayMath.normalize(geFeature2EmpiricalDist[n]);
smoothDistribution(geFeature2EmpiricalDist[n]);
}
// now build the inverted index from each GE feature to unlabeled datums that contain it.
for (int i = 0; i < unlabeledDataList.size(); i++) {
Datum<L, F> datum = unlabeledDataList.get(i);
for (F feature : datum.asFeatures()) {
if (geFeatureMap.containsKey(feature)) {
int geFnum = geFeatureMap.get(feature);
geFeature2DatumList.get(geFnum).add(i);
activeUnlabeledExamples.add(i);
}
}
}
System.out.println("Number of active unlabeled examples:" + activeUnlabeledExamples.size());
}
private void updateDerivative(
Datum<L, F> datum,
double[] probs,
Counter<Triple<Integer, Integer, Integer>> feature2classPairDerivatives) {
for (F feature : datum.asFeatures()) {
int fID = labeledDataset.featureIndex.indexOf(feature);
if (fID >= 0) {
for (int c = 0; c < numClasses; c++) {
for (int cPrime = 0; cPrime < numClasses; cPrime++) {
if (cPrime == c) {
feature2classPairDerivatives.incrementCount(
new Triple<Integer, Integer, Integer>(fID, c, cPrime),
-probs[c] * (1 - probs[c]) * valueOfFeature(feature, datum));
} else {
feature2classPairDerivatives.incrementCount(
new Triple<Integer, Integer, Integer>(fID, c, cPrime),
probs[c] * probs[cPrime] * valueOfFeature(feature, datum));
}
}
}
}
}
}
public Classifier<L, F> trainClassifier(Iterable<Datum<L, F>> dataIterable) {
Minimizer<DiffFunction> minimizer = getMinimizer();
Index<F> featureIndex = Generics.newIndex();
Index<L> labelIndex = Generics.newIndex();
for (Datum<L, F> d : dataIterable) {
labelIndex.add(d.label());
featureIndex.addAll(d.asFeatures()); // If there are duplicates, it doesn't add them again.
}
System.err.println(
String.format(
"Training linear classifier with %d features and %d labels",
featureIndex.size(), labelIndex.size()));
LogConditionalObjectiveFunction<L, F> objective =
new LogConditionalObjectiveFunction<L, F>(dataIterable, logPrior, featureIndex, labelIndex);
objective.setPrior(new LogPrior(LogPrior.LogPriorType.QUADRATIC));
double[] initial = objective.initial();
double[] weights = minimizer.minimize(objective, TOL, initial);
LinearClassifier<L, F> classifier =
new LinearClassifier<L, F>(objective.to2D(weights), featureIndex, labelIndex);
return classifier;
}