@Override
protected VM validateModel(Dataset validationData) {
predictDataset(validationData);
Set<Object> classesSet = knowledgeBase.getModelParameters().getClasses();
// create new validation metrics object
VM validationMetrics = knowledgeBase.getEmptyValidationMetricsObject();
// short notation
Map<List<Object>, Double> ctMap = validationMetrics.getContingencyTable();
for (Object theClass : classesSet) {
ctMap.put(Arrays.<Object>asList(theClass, SensitivityRates.TP), 0.0); // true possitive
ctMap.put(Arrays.<Object>asList(theClass, SensitivityRates.FP), 0.0); // false possitive
ctMap.put(Arrays.<Object>asList(theClass, SensitivityRates.TN), 0.0); // true negative
ctMap.put(Arrays.<Object>asList(theClass, SensitivityRates.FN), 0.0); // false negative
}
int n = validationData.size();
int c = classesSet.size();
int correctCount = 0;
for (Record r : validationData) {
if (r.getYPredicted().equals(r.getY())) {
++correctCount;
for (Object cl : classesSet) {
if (cl.equals(r.getYPredicted())) {
List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.TP);
ctMap.put(tpk, ctMap.get(tpk) + 1.0);
} else {
List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.TN);
ctMap.put(tpk, ctMap.get(tpk) + 1.0);
}
}
} else {
for (Object cl : classesSet) {
if (cl.equals(r.getYPredicted())) {
List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.FP);
ctMap.put(tpk, ctMap.get(tpk) + 1.0);
} else if (cl.equals(r.getY())) {
List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.FN);
ctMap.put(tpk, ctMap.get(tpk) + 1.0);
} else {
List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.TN);
ctMap.put(tpk, ctMap.get(tpk) + 1.0);
}
}
}
}
validationMetrics.setAccuracy(correctCount / (double) n);
// Average Precision, Recall and F1:
// http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.104.8244&rep=rep1&type=pdf
for (Object theClass : classesSet) {
double tp = ctMap.get(Arrays.<Object>asList(theClass, SensitivityRates.TP));
double fp = ctMap.get(Arrays.<Object>asList(theClass, SensitivityRates.FP));
double fn = ctMap.get(Arrays.<Object>asList(theClass, SensitivityRates.FN));
double classPrecision = 0.0;
double classRecall = 0.0;
double classF1 = 0.0;
if (tp > 0.0) {
classPrecision = tp / (tp + fp);
classRecall = tp / (tp + fn);
classF1 = 2.0 * classPrecision * classRecall / (classPrecision + classRecall);
} else if (tp == 0.0 && fp == 0.0 && fn == 0.0) {
// if this category did not appear in the dataset then set the metrics to 1
classPrecision = 1.0;
classRecall = 1.0;
classF1 = 1.0;
}
validationMetrics.getMicroPrecision().put(theClass, classPrecision);
validationMetrics.getMicroRecall().put(theClass, classRecall);
validationMetrics.getMicroF1().put(theClass, classF1);
validationMetrics.setMacroPrecision(
validationMetrics.getMacroPrecision() + classPrecision / c);
validationMetrics.setMacroRecall(validationMetrics.getMacroRecall() + classRecall / c);
validationMetrics.setMacroF1(validationMetrics.getMacroF1() + classF1 / c);
}
return validationMetrics;
}
