/** * Given confusion matrix, it writes it in CSV and LaTeX form to the tasks output directory, and * also prints evaluations (F-measure, Precision, Recall) * * @param context task context * @param confusionMatrix confusion matrix * @param filePrefix prefix of output files * @throws java.io.IOException */ public static void writeOutputResults( TaskContext context, ConfusionMatrix confusionMatrix, String filePrefix) throws IOException { // storing the results as latex confusion matrix String confMatrixFileTex = (filePrefix != null ? filePrefix : "") + "confusionMatrix.tex"; File matrixFolderTex = context.getFolder(Constants.TEST_TASK_OUTPUT_KEY, StorageService.AccessMode.READWRITE); File evaluationFileLaTeX = new File(matrixFolderTex, confMatrixFileTex); FileUtils.writeStringToFile(evaluationFileLaTeX, confusionMatrix.toStringLatex()); // as CSV confusion matrix String confMatrixFileCsv = (filePrefix != null ? filePrefix : "") + "confusionMatrix.csv"; File matrixFolder = context.getFolder(Constants.TEST_TASK_OUTPUT_KEY, StorageService.AccessMode.READWRITE); File evaluationFileCSV = new File(matrixFolder, confMatrixFileCsv); CSVPrinter csvPrinter = new CSVPrinter(new FileWriter(evaluationFileCSV), CSVFormat.DEFAULT); csvPrinter.printRecords(confusionMatrix.toStringMatrix()); IOUtils.closeQuietly(csvPrinter); // and results File evalFolder = context.getFolder(Constants.TEST_TASK_OUTPUT_KEY, StorageService.AccessMode.READWRITE); String evalFileName = new SVMHMMAdapter() .getFrameworkFilename(TCMachineLearningAdapter.AdapterNameEntries.evaluationFile); File evaluationFile = new File(evalFolder, evalFileName); PrintWriter pw = new PrintWriter(evaluationFile); pw.println(confusionMatrix.printNiceResults()); pw.println(confusionMatrix.printLabelPrecRecFm()); pw.println(confusionMatrix.printClassDistributionGold()); IOUtils.closeQuietly(pw); }
private void updateConfusionMatrix( ConfusionMatrix m, Example ex, double compatDecisionThreshold, double probDecisionThreshold) { List<Derivation> derivations = ex.getPredDerivations(); double[] probs = Derivation.getProbs(derivations, 1.0d); for (int i = 0; i < derivations.size(); i++) { Derivation deriv = derivations.get(i); double gold, pred; if (compatDecisionThreshold == -1.0d) gold = deriv.getCompatibility(); else gold = (deriv.getCompatibility() > compatDecisionThreshold) ? 1.0d : 0.0d; if (probDecisionThreshold == -1.0d) pred = probs[i]; else pred = (probs[i] > probDecisionThreshold) ? 1.0d : 0.0d; m.tp += gold * pred; m.fn += gold * (1.0d - pred); m.fp += (1.0d - gold) * pred; m.tn += (1.0d - gold) * (1.0d - pred); } }
public double compute(Rule rule) { double result; ConfusionMatrix confusionMatrix; confusionMatrix = rule.getConfusionMatrix(); result = (confusionMatrix.getTruePositives() / (confusionMatrix.getTruePositives() + confusionMatrix.getFalsePositives()) - (confusionMatrix.getTruePositives() + confusionMatrix.getFalseNegatives()) / confusionMatrix.getNumberOfSamples()) / (1 - ((confusionMatrix.getTruePositives() + confusionMatrix.getFalseNegatives()) / confusionMatrix.getNumberOfSamples())); return result; }
/** * @param correctLabel The correct label * @param classifiedResult The classified result * @return whether the instance was correct or not */ public boolean addInstance(String correctLabel, ClassifierResult classifiedResult) { boolean result = correctLabel.equals(classifiedResult.getLabel()); if (result) { correctlyClassified++; } else { incorrectlyClassified++; } confusionMatrix.addInstance(correctLabel, classifiedResult); if (classifiedResult.getLogLikelihood() != Double.MAX_VALUE) { summarizer.add(classifiedResult.getLogLikelihood()); hasLL = true; } return result; }
@Override public String toString() { StringBuilder returnString = new StringBuilder(); returnString.append('\n'); returnString.append("=======================================================\n"); returnString.append("Summary\n"); returnString.append("-------------------------------------------------------\n"); int totalClassified = correctlyClassified + incorrectlyClassified; double percentageCorrect = (double) 100 * correctlyClassified / totalClassified; double percentageIncorrect = (double) 100 * incorrectlyClassified / totalClassified; NumberFormat decimalFormatter = new DecimalFormat("0.####"); returnString .append(StringUtils.rightPad("Correctly Classified Instances", 40)) .append(": ") .append(StringUtils.leftPad(Integer.toString(correctlyClassified), 10)) .append('\t') .append(StringUtils.leftPad(decimalFormatter.format(percentageCorrect), 10)) .append("%\n"); returnString .append(StringUtils.rightPad("Incorrectly Classified Instances", 40)) .append(": ") .append(StringUtils.leftPad(Integer.toString(incorrectlyClassified), 10)) .append('\t') .append(StringUtils.leftPad(decimalFormatter.format(percentageIncorrect), 10)) .append("%\n"); returnString .append(StringUtils.rightPad("Total Classified Instances", 40)) .append(": ") .append(StringUtils.leftPad(Integer.toString(totalClassified), 10)) .append('\n'); returnString.append('\n'); returnString.append(confusionMatrix); returnString.append("=======================================================\n"); returnString.append("Statistics\n"); returnString.append("-------------------------------------------------------\n"); RunningAverageAndStdDev normStats = confusionMatrix.getNormalizedStats(); returnString .append(StringUtils.rightPad("Kappa", 40)) .append(StringUtils.leftPad(decimalFormatter.format(confusionMatrix.getKappa()), 10)) .append('\n'); returnString .append(StringUtils.rightPad("Accuracy", 40)) .append(StringUtils.leftPad(decimalFormatter.format(confusionMatrix.getAccuracy()), 10)) .append("%\n"); returnString .append(StringUtils.rightPad("Reliability", 40)) .append( StringUtils.leftPad(decimalFormatter.format(normStats.getAverage() * 100.00000001), 10)) .append("%\n"); returnString .append(StringUtils.rightPad("Reliability (standard deviation)", 40)) .append(StringUtils.leftPad(decimalFormatter.format(normStats.getStandardDeviation()), 10)) .append('\n'); returnString .append(StringUtils.rightPad("Weighted precision", 40)) .append( StringUtils.leftPad( decimalFormatter.format(confusionMatrix.getWeightedPrecision()), 10)) .append('\n'); returnString .append(StringUtils.rightPad("Weighted recall", 40)) .append( StringUtils.leftPad(decimalFormatter.format(confusionMatrix.getWeightedRecall()), 10)) .append('\n'); returnString .append(StringUtils.rightPad("Weighted F1 score", 40)) .append( StringUtils.leftPad(decimalFormatter.format(confusionMatrix.getWeightedF1score()), 10)) .append('\n'); if (hasLL) { returnString .append(StringUtils.rightPad("Log-likelihood", 30)) .append("mean : ") .append(StringUtils.leftPad(decimalFormatter.format(summarizer.getMean()), 10)) .append('\n'); returnString .append(StringUtils.rightPad("", 30)) .append(StringUtils.rightPad("25%-ile : ", 10)) .append(StringUtils.leftPad(decimalFormatter.format(summarizer.getQuartile(1)), 10)) .append('\n'); returnString .append(StringUtils.rightPad("", 30)) .append(StringUtils.rightPad("75%-ile : ", 10)) .append(StringUtils.leftPad(decimalFormatter.format(summarizer.getQuartile(3)), 10)) .append('\n'); } return returnString.toString(); }
/** Test of evaluatePerformance method, of class ConfusionMatrixPerformanceEvaluator. */ @Test public void testEvaluatePerformance() { double epsilon = 1e-10; ConfusionMatrixPerformanceEvaluator<?, String> instance = new ConfusionMatrixPerformanceEvaluator<String, String>(); Collection<TargetEstimatePair<String, String>> data = new ArrayList<TargetEstimatePair<String, String>>(); ConfusionMatrix<String> confusion = instance.summarize(data); assertEquals(0.0, confusion.getTotalCount(), 0.0); data.add(new DefaultTargetEstimatePair<String, String>("yes", "no")); confusion = instance.summarize(data); assertEquals(1.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0 / 1.0, confusion.getErrorRate(), 0.0); data.add(new DefaultTargetEstimatePair<String, String>("yes", "yes")); confusion = instance.summarize(data); assertEquals(2.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0 / 2.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>("no", "no")); confusion = instance.summarize(data); assertEquals(3.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0 / 3.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>("something", "else")); confusion = instance.summarize(data); assertEquals(4.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0, confusion.getCount("something", "else"), 0.0); assertEquals(2.0 / 4.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>("same", "same")); confusion = instance.summarize(data); assertEquals(5.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0, confusion.getCount("something", "else"), 0.0); assertEquals(1.0, confusion.getCount("same", "same"), 0.0); assertEquals(2.0 / 5.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>("oh", "no")); confusion = instance.summarize(data); assertEquals(6.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0, confusion.getCount("something", "else"), 0.0); assertEquals(1.0, confusion.getCount("same", "same"), 0.0); assertEquals(1.0, confusion.getCount("oh", "no"), 0.0); assertEquals(3.0 / 6.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>("this", "bad")); confusion = instance.summarize(data); assertEquals(7.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0, confusion.getCount("something", "else"), 0.0); assertEquals(1.0, confusion.getCount("same", "same"), 0.0); assertEquals(1.0, confusion.getCount("oh", "no"), 0.0); assertEquals(1.0, confusion.getCount("this", "bad"), 0.0); assertEquals(4.0 / 7.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>("not null", null)); confusion = instance.summarize(data); assertEquals(8.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0, confusion.getCount("something", "else"), 0.0); assertEquals(1.0, confusion.getCount("same", "same"), 0.0); assertEquals(1.0, confusion.getCount("oh", "no"), 0.0); assertEquals(1.0, confusion.getCount("this", "bad"), 0.0); assertEquals(1.0, confusion.getCount("not null", null), 0.0); assertEquals(5.0 / 8.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>(null, "not null")); confusion = instance.summarize(data); assertEquals(9.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0, confusion.getCount("something", "else"), 0.0); assertEquals(1.0, confusion.getCount("same", "same"), 0.0); assertEquals(1.0, confusion.getCount("oh", "no"), 0.0); assertEquals(1.0, confusion.getCount("this", "bad"), 0.0); assertEquals(1.0, confusion.getCount("not null", null), 0.0); assertEquals(1.0, confusion.getCount(null, "not null"), 0.0); assertEquals(6.0 / 9.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>(null, null)); confusion = instance.summarize(data); assertEquals(10.0, confusion.getTotalCount(), 0.0); assertEquals(1.0, confusion.getCount("yes", "no"), 0.0); assertEquals(1.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(1.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0, confusion.getCount("something", "else"), 0.0); assertEquals(1.0, confusion.getCount("same", "same"), 0.0); assertEquals(1.0, confusion.getCount("oh", "no"), 0.0); assertEquals(1.0, confusion.getCount("this", "bad"), 0.0); assertEquals(1.0, confusion.getCount("not null", null), 0.0); assertEquals(1.0, confusion.getCount(null, "not null"), 0.0); assertEquals(1.0, confusion.getCount(null, null), 0.0); assertEquals(6.0 / 10.0, confusion.getErrorRate(), epsilon); data.add(new DefaultTargetEstimatePair<String, String>("yes", "no")); data.add(new DefaultTargetEstimatePair<String, String>("yes", "no")); data.add(new DefaultTargetEstimatePair<String, String>("yes", "yes")); data.add(new DefaultTargetEstimatePair<String, String>("no", "no")); confusion = instance.summarize(data); assertEquals(14.0, confusion.getTotalCount(), 0.0); assertEquals(3.0, confusion.getCount("yes", "no"), 0.0); assertEquals(2.0, confusion.getCount("yes", "yes"), 0.0); assertEquals(2.0, confusion.getCount("no", "no"), 0.0); assertEquals(1.0, confusion.getCount("something", "else"), 0.0); assertEquals(1.0, confusion.getCount("same", "same"), 0.0); assertEquals(1.0, confusion.getCount("oh", "no"), 0.0); assertEquals(1.0, confusion.getCount("this", "bad"), 0.0); assertEquals(1.0, confusion.getCount("not null", null), 0.0); assertEquals(1.0, confusion.getCount(null, "not null"), 0.0); assertEquals(1.0, confusion.getCount(null, null), 0.0); assertEquals(8.0 / 14.0, confusion.getErrorRate(), epsilon); }
protected double doScoringAndSaveModel( boolean finalScoring, boolean oob, boolean build_tree_one_node) { double training_r2 = Double.NaN; // Training R^2 value, if computed long now = System.currentTimeMillis(); if (_firstScore == 0) _firstScore = now; long sinceLastScore = now - _timeLastScoreStart; boolean updated = false; new ProgressUpdate( "Built " + _model._output._ntrees + " trees so far (out of " + _parms._ntrees + ").") .fork(_progressKey); // Now model already contains tid-trees in serialized form if (_parms._score_each_iteration || finalScoring || (now - _firstScore < 4000) || // Score every time for 4 secs // Throttle scoring to keep the cost sane; limit to a 10% duty cycle & every 4 secs (sinceLastScore > 4000 && // Limit scoring updates to every 4sec (double) (_timeLastScoreEnd - _timeLastScoreStart) / sinceLastScore < 0.1)) { // 10% duty cycle checkMemoryFootPrint(); // If validation is specified we use a model for scoring, so we need to // update it! First we save model with trees (i.e., make them available // for scoring) and then update it with resulting error _model.update(_key); updated = true; Log.info("============================================================== "); SharedTreeModel.SharedTreeOutput out = _model._output; _timeLastScoreStart = now; // Score on training data new ProgressUpdate("Scoring the model.").fork(_progressKey); Score sc = new Score(this, true, oob, _model._output.getModelCategory()) .doAll(train(), build_tree_one_node); ModelMetrics mm = sc.makeModelMetrics(_model, _parms.train()); out._training_metrics = mm; if (oob) out._training_metrics._description = "Metrics reported on Out-Of-Bag training samples"; out._scored_train[out._ntrees].fillFrom(mm); if (out._ntrees > 0) Log.info("Training " + out._scored_train[out._ntrees].toString()); // Score again on validation data if (_parms._valid != null) { Score scv = new Score(this, false, false, _model._output.getModelCategory()) .doAll(valid(), build_tree_one_node); ModelMetrics mmv = scv.makeModelMetrics(_model, _parms.valid()); out._validation_metrics = mmv; out._scored_valid[out._ntrees].fillFrom(mmv); if (out._ntrees > 0) Log.info("Validation " + out._scored_valid[out._ntrees].toString()); } if (out._ntrees > 0) { // Compute variable importances out._model_summary = createModelSummaryTable(out); out._scoring_history = createScoringHistoryTable(out); out._varimp = new hex.VarImp(_improvPerVar, out._names); out._variable_importances = hex.ModelMetrics.calcVarImp(out._varimp); Log.info(out._model_summary.toString()); // For Debugging: // Log.info(out._scoring_history.toString()); // Log.info(out._variable_importances.toString()); } ConfusionMatrix cm = mm.cm(); if (cm != null) { if (cm._cm.length <= _parms._max_confusion_matrix_size) { Log.info(cm.toASCII()); } else { Log.info( "Confusion Matrix is too large (max_confusion_matrix_size=" + _parms._max_confusion_matrix_size + "): " + _nclass + " classes."); } } _timeLastScoreEnd = System.currentTimeMillis(); } // Double update - after either scoring or variable importance if (updated) _model.update(_key); return training_r2; }