protected static Evaluation adjustForInputMappedClassifier(
Evaluation eval,
weka.classifiers.Classifier classifier,
Instances inst,
ClassifierErrorsPlotInstances plotInstances)
throws Exception {
if (classifier instanceof weka.classifiers.misc.InputMappedClassifier) {
Instances mappedClassifierHeader =
((weka.classifiers.misc.InputMappedClassifier) classifier)
.getModelHeader(new Instances(inst, 0));
eval = new Evaluation(new Instances(mappedClassifierHeader, 0));
if (!eval.getHeader().equalHeaders(inst)) {
// When the InputMappedClassifier is loading a model,
// we need to make a new dataset that maps the test instances to
// the structure expected by the mapped classifier - this is only
// to ensure that the ClassifierPlotInstances object is configured
// in accordance with what the embeded classifier was trained with
Instances mappedClassifierDataset =
((weka.classifiers.misc.InputMappedClassifier) classifier)
.getModelHeader(new Instances(mappedClassifierHeader, 0));
for (int zz = 0; zz < inst.numInstances(); zz++) {
Instance mapped =
((weka.classifiers.misc.InputMappedClassifier) classifier)
.constructMappedInstance(inst.instance(zz));
mappedClassifierDataset.add(mapped);
}
eval.setPriors(mappedClassifierDataset);
plotInstances.setInstances(mappedClassifierDataset);
plotInstances.setClassifier(classifier);
plotInstances.setClassIndex(mappedClassifierDataset.classIndex());
plotInstances.setEvaluation(eval);
}
}
return eval;
}
/**
* Accept a classifier to be evaluated.
*
* @param ce a <code>BatchClassifierEvent</code> value
*/
public void acceptClassifier(BatchClassifierEvent ce) {
if (ce.getTestSet() == null || ce.getTestSet().isStructureOnly()) {
return; // can't evaluate empty/non-existent test instances
}
Classifier classifier = ce.getClassifier();
try {
if (ce.getGroupIdentifier() != m_currentBatchIdentifier) {
if (m_setsComplete > 0) {
if (m_logger != null) {
m_logger.statusMessage(
statusMessagePrefix() + "BUSY. Can't accept data " + "at this time.");
m_logger.logMessage(
"[ClassifierPerformanceEvaluator] "
+ statusMessagePrefix()
+ " BUSY. Can't accept data at this time.");
}
return;
}
if (ce.getTrainSet().getDataSet() == null
|| ce.getTrainSet().getDataSet().numInstances() == 0) {
// we have no training set to estimate majority class
// or mean of target from
Evaluation eval = new Evaluation(ce.getTestSet().getDataSet());
m_PlotInstances = ExplorerDefaults.getClassifierErrorsPlotInstances();
m_PlotInstances.setInstances(ce.getTestSet().getDataSet());
m_PlotInstances.setClassifier(ce.getClassifier());
m_PlotInstances.setClassIndex(ce.getTestSet().getDataSet().classIndex());
m_PlotInstances.setEvaluation(eval);
eval =
adjustForInputMappedClassifier(
eval, ce.getClassifier(), ce.getTestSet().getDataSet(), m_PlotInstances);
eval.useNoPriors();
m_eval = new AggregateableEvaluation(eval);
} else {
// we can set up with the training set here
Evaluation eval = new Evaluation(ce.getTrainSet().getDataSet());
m_PlotInstances = ExplorerDefaults.getClassifierErrorsPlotInstances();
m_PlotInstances.setInstances(ce.getTrainSet().getDataSet());
m_PlotInstances.setClassifier(ce.getClassifier());
m_PlotInstances.setClassIndex(ce.getTestSet().getDataSet().classIndex());
m_PlotInstances.setEvaluation(eval);
eval =
adjustForInputMappedClassifier(
eval, ce.getClassifier(), ce.getTrainSet().getDataSet(), m_PlotInstances);
m_eval = new AggregateableEvaluation(eval);
}
m_PlotInstances.setUp();
m_currentBatchIdentifier = ce.getGroupIdentifier();
m_setsComplete = 0;
m_aggregatedPlotInstances = null;
String msg =
"[ClassifierPerformanceEvaluator] "
+ statusMessagePrefix()
+ " starting executor pool ("
+ getExecutionSlots()
+ " slots)...";
// start the execution pool
if (m_executorPool == null) {
startExecutorPool();
}
m_tasks = new ArrayList<EvaluationTask>();
if (m_logger != null) {
m_logger.logMessage(msg);
} else {
System.out.println(msg);
}
}
// if m_tasks == null then we've been stopped
if (m_setsComplete < ce.getMaxSetNumber() && m_tasks != null) {
EvaluationTask newTask =
new EvaluationTask(
classifier,
ce.getTrainSet().getDataSet(),
ce.getTestSet().getDataSet(),
ce.getSetNumber(),
ce.getMaxSetNumber());
String msg =
"[ClassifierPerformanceEvaluator] "
+ statusMessagePrefix()
+ " scheduling "
+ " evaluation of fold "
+ ce.getSetNumber()
+ " for execution...";
if (m_logger != null) {
m_logger.logMessage(msg);
} else {
System.out.println(msg);
}
m_tasks.add(newTask);
m_executorPool.execute(newTask);
}
} catch (Exception ex) {
// stop everything
stop();
}
}
public void execute() {
if (m_stopped) {
return;
}
if (m_logger != null) {
m_logger.statusMessage(statusMessagePrefix() + "Evaluating (" + m_setNum + ")...");
m_visual.setAnimated();
}
try {
ClassifierErrorsPlotInstances plotInstances =
ExplorerDefaults.getClassifierErrorsPlotInstances();
Evaluation eval = null;
if (m_trainData == null || m_trainData.numInstances() == 0) {
eval = new Evaluation(m_testData);
plotInstances.setInstances(m_testData);
plotInstances.setClassifier(m_classifier);
plotInstances.setClassIndex(m_testData.classIndex());
plotInstances.setEvaluation(eval);
eval = adjustForInputMappedClassifier(eval, m_classifier, m_testData, plotInstances);
eval.useNoPriors();
} else {
eval = new Evaluation(m_trainData);
plotInstances.setInstances(m_trainData);
plotInstances.setClassifier(m_classifier);
plotInstances.setClassIndex(m_trainData.classIndex());
plotInstances.setEvaluation(eval);
eval = adjustForInputMappedClassifier(eval, m_classifier, m_trainData, plotInstances);
}
plotInstances.setUp();
for (int i = 0; i < m_testData.numInstances(); i++) {
if (m_stopped) {
break;
}
Instance temp = m_testData.instance(i);
plotInstances.process(temp, m_classifier, eval);
}
if (m_stopped) {
return;
}
aggregateEvalTask(eval, m_classifier, m_testData, plotInstances, m_setNum, m_maxSetNum);
} catch (Exception ex) {
ClassifierPerformanceEvaluator.this.stop(); // stop all processing
if (m_logger != null) {
m_logger.logMessage(
"[ClassifierPerformanceEvaluator] "
+ statusMessagePrefix()
+ " problem evaluating classifier. "
+ ex.getMessage());
}
ex.printStackTrace();
}
}