@Override protected VM validateModel(Dataset validationData) { predictDataset(validationData); Set<Object> classesSet = knowledgeBase.getModelParameters().getClasses(); // create new validation metrics object VM validationMetrics = knowledgeBase.getEmptyValidationMetricsObject(); // short notation Map<List<Object>, Double> ctMap = validationMetrics.getContingencyTable(); for (Object theClass : classesSet) { ctMap.put(Arrays.<Object>asList(theClass, SensitivityRates.TP), 0.0); // true possitive ctMap.put(Arrays.<Object>asList(theClass, SensitivityRates.FP), 0.0); // false possitive ctMap.put(Arrays.<Object>asList(theClass, SensitivityRates.TN), 0.0); // true negative ctMap.put(Arrays.<Object>asList(theClass, SensitivityRates.FN), 0.0); // false negative } int n = validationData.size(); int c = classesSet.size(); int correctCount = 0; for (Record r : validationData) { if (r.getYPredicted().equals(r.getY())) { ++correctCount; for (Object cl : classesSet) { if (cl.equals(r.getYPredicted())) { List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.TP); ctMap.put(tpk, ctMap.get(tpk) + 1.0); } else { List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.TN); ctMap.put(tpk, ctMap.get(tpk) + 1.0); } } } else { for (Object cl : classesSet) { if (cl.equals(r.getYPredicted())) { List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.FP); ctMap.put(tpk, ctMap.get(tpk) + 1.0); } else if (cl.equals(r.getY())) { List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.FN); ctMap.put(tpk, ctMap.get(tpk) + 1.0); } else { List<Object> tpk = Arrays.<Object>asList(cl, SensitivityRates.TN); ctMap.put(tpk, ctMap.get(tpk) + 1.0); } } } } validationMetrics.setAccuracy(correctCount / (double) n); // Average Precision, Recall and F1: // http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.104.8244&rep=rep1&type=pdf for (Object theClass : classesSet) { double tp = ctMap.get(Arrays.<Object>asList(theClass, SensitivityRates.TP)); double fp = ctMap.get(Arrays.<Object>asList(theClass, SensitivityRates.FP)); double fn = ctMap.get(Arrays.<Object>asList(theClass, SensitivityRates.FN)); double classPrecision = 0.0; double classRecall = 0.0; double classF1 = 0.0; if (tp > 0.0) { classPrecision = tp / (tp + fp); classRecall = tp / (tp + fn); classF1 = 2.0 * classPrecision * classRecall / (classPrecision + classRecall); } else if (tp == 0.0 && fp == 0.0 && fn == 0.0) { // if this category did not appear in the dataset then set the metrics to 1 classPrecision = 1.0; classRecall = 1.0; classF1 = 1.0; } validationMetrics.getMicroPrecision().put(theClass, classPrecision); validationMetrics.getMicroRecall().put(theClass, classRecall); validationMetrics.getMicroF1().put(theClass, classF1); validationMetrics.setMacroPrecision( validationMetrics.getMacroPrecision() + classPrecision / c); validationMetrics.setMacroRecall(validationMetrics.getMacroRecall() + classRecall / c); validationMetrics.setMacroF1(validationMetrics.getMacroF1() + classF1 / c); } return validationMetrics; }