@Override
protected pikater.ontology.messages.Evaluation evaluateCA() {
Evaluation eval = test();
pikater.ontology.messages.Evaluation result = new pikater.ontology.messages.Evaluation();
result.setError_rate((float) eval.errorRate());
try {
result.setKappa_statistic((float) eval.kappa());
} catch (Exception e) {
result.setKappa_statistic(-1);
}
result.setMean_absolute_error((float) eval.meanAbsoluteError());
try {
result.setRelative_absolute_error((float) eval.relativeAbsoluteError());
} catch (Exception e) {
result.setRelative_absolute_error(-1);
}
result.setRoot_mean_squared_error((float) eval.rootMeanSquaredError());
result.setRoot_relative_squared_error((float) eval.rootRelativeSquaredError());
return result;
}
/**
* Accepts and processes a classifier encapsulated in an incremental classifier event
*
* @param ce an <code>IncrementalClassifierEvent</code> value
*/
@Override
public void acceptClassifier(final IncrementalClassifierEvent ce) {
try {
if (ce.getStatus() == IncrementalClassifierEvent.NEW_BATCH) {
m_throughput = new StreamThroughput(statusMessagePrefix());
m_throughput.setSamplePeriod(m_statusFrequency);
// m_eval = new Evaluation(ce.getCurrentInstance().dataset());
m_eval = new Evaluation(ce.getStructure());
m_eval.useNoPriors();
m_dataLegend = new Vector();
m_reset = true;
m_dataPoint = new double[0];
Instances inst = ce.getStructure();
System.err.println("NEW BATCH");
m_instanceCount = 0;
if (m_windowSize > 0) {
m_window = new LinkedList<Instance>();
m_windowEval = new Evaluation(ce.getStructure());
m_windowEval.useNoPriors();
m_windowedPreds = new LinkedList<double[]>();
if (m_logger != null) {
m_logger.logMessage(
statusMessagePrefix()
+ "[IncrementalClassifierEvaluator] Chart output using windowed "
+ "evaluation over "
+ m_windowSize
+ " instances");
}
}
/*
* if (m_logger != null) { m_logger.statusMessage(statusMessagePrefix()
* + "IncrementalClassifierEvaluator: started processing...");
* m_logger.logMessage(statusMessagePrefix() +
* " [IncrementalClassifierEvaluator]" + statusMessagePrefix() +
* " started processing..."); }
*/
} else {
Instance inst = ce.getCurrentInstance();
if (inst != null) {
m_throughput.updateStart();
m_instanceCount++;
// if (inst.attribute(inst.classIndex()).isNominal()) {
double[] dist = ce.getClassifier().distributionForInstance(inst);
double pred = 0;
if (!inst.isMissing(inst.classIndex())) {
if (m_outputInfoRetrievalStats) {
// store predictions so AUC etc can be output.
m_eval.evaluateModelOnceAndRecordPrediction(dist, inst);
} else {
m_eval.evaluateModelOnce(dist, inst);
}
if (m_windowSize > 0) {
m_windowEval.evaluateModelOnce(dist, inst);
m_window.addFirst(inst);
m_windowedPreds.addFirst(dist);
if (m_instanceCount > m_windowSize) {
// "forget" the oldest prediction
Instance oldest = m_window.removeLast();
double[] oldDist = m_windowedPreds.removeLast();
oldest.setWeight(-oldest.weight());
m_windowEval.evaluateModelOnce(oldDist, oldest);
oldest.setWeight(-oldest.weight());
}
}
} else {
pred = ce.getClassifier().classifyInstance(inst);
}
if (inst.classIndex() >= 0) {
// need to check that the class is not missing
if (inst.attribute(inst.classIndex()).isNominal()) {
if (!inst.isMissing(inst.classIndex())) {
if (m_dataPoint.length < 2) {
m_dataPoint = new double[3];
m_dataLegend.addElement("Accuracy");
m_dataLegend.addElement("RMSE (prob)");
m_dataLegend.addElement("Kappa");
}
// int classV = (int) inst.value(inst.classIndex());
if (m_windowSize > 0) {
m_dataPoint[1] = m_windowEval.rootMeanSquaredError();
m_dataPoint[2] = m_windowEval.kappa();
} else {
m_dataPoint[1] = m_eval.rootMeanSquaredError();
m_dataPoint[2] = m_eval.kappa();
}
// int maxO = Utils.maxIndex(dist);
// if (maxO == classV) {
// dist[classV] = -1;
// maxO = Utils.maxIndex(dist);
// }
// m_dataPoint[1] -= dist[maxO];
} else {
if (m_dataPoint.length < 1) {
m_dataPoint = new double[1];
m_dataLegend.addElement("Confidence");
}
}
double primaryMeasure = 0;
if (!inst.isMissing(inst.classIndex())) {
if (m_windowSize > 0) {
primaryMeasure = 1.0 - m_windowEval.errorRate();
} else {
primaryMeasure = 1.0 - m_eval.errorRate();
}
} else {
// record confidence as the primary measure
// (another possibility would be entropy of
// the distribution, or perhaps average
// confidence)
primaryMeasure = dist[Utils.maxIndex(dist)];
}
// double [] dataPoint = new double[1];
m_dataPoint[0] = primaryMeasure;
// double min = 0; double max = 100;
/*
* ChartEvent e = new
* ChartEvent(IncrementalClassifierEvaluator.this, m_dataLegend,
* min, max, dataPoint);
*/
m_ce.setLegendText(m_dataLegend);
m_ce.setMin(0);
m_ce.setMax(1);
m_ce.setDataPoint(m_dataPoint);
m_ce.setReset(m_reset);
m_reset = false;
} else {
// numeric class
if (m_dataPoint.length < 1) {
m_dataPoint = new double[1];
if (inst.isMissing(inst.classIndex())) {
m_dataLegend.addElement("Prediction");
} else {
m_dataLegend.addElement("RMSE");
}
}
if (!inst.isMissing(inst.classIndex())) {
double update;
if (!inst.isMissing(inst.classIndex())) {
if (m_windowSize > 0) {
update = m_windowEval.rootMeanSquaredError();
} else {
update = m_eval.rootMeanSquaredError();
}
} else {
update = pred;
}
m_dataPoint[0] = update;
if (update > m_max) {
m_max = update;
}
if (update < m_min) {
m_min = update;
}
}
m_ce.setLegendText(m_dataLegend);
m_ce.setMin((inst.isMissing(inst.classIndex()) ? m_min : 0));
m_ce.setMax(m_max);
m_ce.setDataPoint(m_dataPoint);
m_ce.setReset(m_reset);
m_reset = false;
}
notifyChartListeners(m_ce);
}
m_throughput.updateEnd(m_logger);
}
if (ce.getStatus() == IncrementalClassifierEvent.BATCH_FINISHED || inst == null) {
if (m_logger != null) {
m_logger.logMessage(
"[IncrementalClassifierEvaluator]"
+ statusMessagePrefix()
+ " Finished processing.");
}
m_throughput.finished(m_logger);
// save memory if using windowed evaluation for charting
m_windowEval = null;
m_window = null;
m_windowedPreds = null;
if (m_textListeners.size() > 0) {
String textTitle = ce.getClassifier().getClass().getName();
textTitle = textTitle.substring(textTitle.lastIndexOf('.') + 1, textTitle.length());
String results =
"=== Performance information ===\n\n"
+ "Scheme: "
+ textTitle
+ "\n"
+ "Relation: "
+ m_eval.getHeader().relationName()
+ "\n\n"
+ m_eval.toSummaryString();
if (m_eval.getHeader().classIndex() >= 0
&& m_eval.getHeader().classAttribute().isNominal()
&& (m_outputInfoRetrievalStats)) {
results += "\n" + m_eval.toClassDetailsString();
}
if (m_eval.getHeader().classIndex() >= 0
&& m_eval.getHeader().classAttribute().isNominal()) {
results += "\n" + m_eval.toMatrixString();
}
textTitle = "Results: " + textTitle;
TextEvent te = new TextEvent(this, results, textTitle);
notifyTextListeners(te);
}
}
}
} catch (Exception ex) {
if (m_logger != null) {
m_logger.logMessage(
"[IncrementalClassifierEvaluator]"
+ statusMessagePrefix()
+ " Error processing prediction "
+ ex.getMessage());
m_logger.statusMessage(
statusMessagePrefix() + "ERROR: problem processing prediction (see log for details)");
}
ex.printStackTrace();
stop();
}
}
/**
* Evaluates a feature subset by cross validation
*
* @param feature_set the subset to be evaluated
* @param num_atts the number of attributes in the subset
* @return the estimated accuracy
* @throws Exception if subset can't be evaluated
*/
protected double estimatePerformance(BitSet feature_set, int num_atts) throws Exception {
m_evaluation = new Evaluation(m_theInstances);
int i;
int[] fs = new int[num_atts];
double[] instA = new double[num_atts];
int classI = m_theInstances.classIndex();
int index = 0;
for (i = 0; i < m_numAttributes; i++) {
if (feature_set.get(i)) {
fs[index++] = i;
}
}
// create new hash table
m_entries = new Hashtable((int) (m_theInstances.numInstances() * 1.5));
// insert instances into the hash table
for (i = 0; i < m_numInstances; i++) {
Instance inst = m_theInstances.instance(i);
for (int j = 0; j < fs.length; j++) {
if (fs[j] == classI) {
instA[j] = Double.MAX_VALUE; // missing for the class
} else if (inst.isMissing(fs[j])) {
instA[j] = Double.MAX_VALUE;
} else {
instA[j] = inst.value(fs[j]);
}
}
insertIntoTable(inst, instA);
}
if (m_CVFolds == 1) {
// calculate leave one out error
for (i = 0; i < m_numInstances; i++) {
Instance inst = m_theInstances.instance(i);
for (int j = 0; j < fs.length; j++) {
if (fs[j] == classI) {
instA[j] = Double.MAX_VALUE; // missing for the class
} else if (inst.isMissing(fs[j])) {
instA[j] = Double.MAX_VALUE;
} else {
instA[j] = inst.value(fs[j]);
}
}
evaluateInstanceLeaveOneOut(inst, instA);
}
} else {
m_theInstances.randomize(m_rr);
m_theInstances.stratify(m_CVFolds);
// calculate 10 fold cross validation error
for (i = 0; i < m_CVFolds; i++) {
Instances insts = m_theInstances.testCV(m_CVFolds, i);
evaluateFoldCV(insts, fs);
}
}
switch (m_evaluationMeasure) {
case EVAL_DEFAULT:
if (m_classIsNominal) {
return m_evaluation.pctCorrect();
}
return -m_evaluation.rootMeanSquaredError();
case EVAL_ACCURACY:
return m_evaluation.pctCorrect();
case EVAL_RMSE:
return -m_evaluation.rootMeanSquaredError();
case EVAL_MAE:
return -m_evaluation.meanAbsoluteError();
case EVAL_AUC:
double[] classPriors = m_evaluation.getClassPriors();
Utils.normalize(classPriors);
double weightedAUC = 0;
for (i = 0; i < m_theInstances.classAttribute().numValues(); i++) {
double tempAUC = m_evaluation.areaUnderROC(i);
if (!Utils.isMissingValue(tempAUC)) {
weightedAUC += (classPriors[i] * tempAUC);
} else {
System.err.println("Undefined AUC!!");
}
}
return weightedAUC;
}
// shouldn't get here
return 0.0;
}
/**
* Gets the results for the supplied train and test datasets. Now performs a deep copy of the
* classifier before it is built and evaluated (just in case the classifier is not initialized
* properly in buildClassifier()).
*
* @param train the training Instances.
* @param test the testing Instances.
* @return the results stored in an array. The objects stored in the array may be Strings,
* Doubles, or null (for the missing value).
* @throws Exception if a problem occurs while getting the results
*/
public Object[] getResult(Instances train, Instances test) throws Exception {
if (train.classAttribute().type() != Attribute.NUMERIC) {
throw new Exception("Class attribute is not numeric!");
}
if (m_Template == null) {
throw new Exception("No classifier has been specified");
}
ThreadMXBean thMonitor = ManagementFactory.getThreadMXBean();
boolean canMeasureCPUTime = thMonitor.isThreadCpuTimeSupported();
if (canMeasureCPUTime && !thMonitor.isThreadCpuTimeEnabled())
thMonitor.setThreadCpuTimeEnabled(true);
int addm = (m_AdditionalMeasures != null) ? m_AdditionalMeasures.length : 0;
Object[] result = new Object[RESULT_SIZE + addm + m_numPluginStatistics];
long thID = Thread.currentThread().getId();
long CPUStartTime = -1,
trainCPUTimeElapsed = -1,
testCPUTimeElapsed = -1,
trainTimeStart,
trainTimeElapsed,
testTimeStart,
testTimeElapsed;
Evaluation eval = new Evaluation(train);
m_Classifier = AbstractClassifier.makeCopy(m_Template);
trainTimeStart = System.currentTimeMillis();
if (canMeasureCPUTime) CPUStartTime = thMonitor.getThreadUserTime(thID);
m_Classifier.buildClassifier(train);
if (canMeasureCPUTime) trainCPUTimeElapsed = thMonitor.getThreadUserTime(thID) - CPUStartTime;
trainTimeElapsed = System.currentTimeMillis() - trainTimeStart;
testTimeStart = System.currentTimeMillis();
if (canMeasureCPUTime) CPUStartTime = thMonitor.getThreadUserTime(thID);
eval.evaluateModel(m_Classifier, test);
if (canMeasureCPUTime) testCPUTimeElapsed = thMonitor.getThreadUserTime(thID) - CPUStartTime;
testTimeElapsed = System.currentTimeMillis() - testTimeStart;
thMonitor = null;
m_result = eval.toSummaryString();
// The results stored are all per instance -- can be multiplied by the
// number of instances to get absolute numbers
int current = 0;
result[current++] = new Double(train.numInstances());
result[current++] = new Double(eval.numInstances());
result[current++] = new Double(eval.meanAbsoluteError());
result[current++] = new Double(eval.rootMeanSquaredError());
result[current++] = new Double(eval.relativeAbsoluteError());
result[current++] = new Double(eval.rootRelativeSquaredError());
result[current++] = new Double(eval.correlationCoefficient());
result[current++] = new Double(eval.SFPriorEntropy());
result[current++] = new Double(eval.SFSchemeEntropy());
result[current++] = new Double(eval.SFEntropyGain());
result[current++] = new Double(eval.SFMeanPriorEntropy());
result[current++] = new Double(eval.SFMeanSchemeEntropy());
result[current++] = new Double(eval.SFMeanEntropyGain());
// Timing stats
result[current++] = new Double(trainTimeElapsed / 1000.0);
result[current++] = new Double(testTimeElapsed / 1000.0);
if (canMeasureCPUTime) {
result[current++] = new Double((trainCPUTimeElapsed / 1000000.0) / 1000.0);
result[current++] = new Double((testCPUTimeElapsed / 1000000.0) / 1000.0);
} else {
result[current++] = new Double(Utils.missingValue());
result[current++] = new Double(Utils.missingValue());
}
// sizes
if (m_NoSizeDetermination) {
result[current++] = -1.0;
result[current++] = -1.0;
result[current++] = -1.0;
} else {
ByteArrayOutputStream bastream = new ByteArrayOutputStream();
ObjectOutputStream oostream = new ObjectOutputStream(bastream);
oostream.writeObject(m_Classifier);
result[current++] = new Double(bastream.size());
bastream = new ByteArrayOutputStream();
oostream = new ObjectOutputStream(bastream);
oostream.writeObject(train);
result[current++] = new Double(bastream.size());
bastream = new ByteArrayOutputStream();
oostream = new ObjectOutputStream(bastream);
oostream.writeObject(test);
result[current++] = new Double(bastream.size());
}
// Prediction interval statistics
result[current++] = new Double(eval.coverageOfTestCasesByPredictedRegions());
result[current++] = new Double(eval.sizeOfPredictedRegions());
if (m_Classifier instanceof Summarizable) {
result[current++] = ((Summarizable) m_Classifier).toSummaryString();
} else {
result[current++] = null;
}
for (int i = 0; i < addm; i++) {
if (m_doesProduce[i]) {
try {
double dv =
((AdditionalMeasureProducer) m_Classifier).getMeasure(m_AdditionalMeasures[i]);
if (!Utils.isMissingValue(dv)) {
Double value = new Double(dv);
result[current++] = value;
} else {
result[current++] = null;
}
} catch (Exception ex) {
System.err.println(ex);
}
} else {
result[current++] = null;
}
}
// get the actual metrics from the evaluation object
List<AbstractEvaluationMetric> metrics = eval.getPluginMetrics();
if (metrics != null) {
for (AbstractEvaluationMetric m : metrics) {
if (m.appliesToNumericClass()) {
List<String> statNames = m.getStatisticNames();
for (String s : statNames) {
result[current++] = new Double(m.getStatistic(s));
}
}
}
}
if (current != RESULT_SIZE + addm + m_numPluginStatistics) {
throw new Error("Results didn't fit RESULT_SIZE");
}
return result;
}
