/** Updates the test chooser with possible tests */
protected void setTTester() {
String name = (new SimpleDateFormat("HH:mm:ss - ")).format(new Date()) + "Available resultsets";
StringBuffer outBuff = new StringBuffer();
outBuff.append("Available resultsets\n" + m_TTester.resultsetKey() + "\n\n");
m_History.addResult(name, outBuff);
m_History.setSingle(name);
m_TestsModel.removeAllElements();
for (int i = 0; i < m_TTester.getNumResultsets(); i++) {
String tname = m_TTester.getResultsetName(i);
/* if (tname.length() > 20) {
tname = tname.substring(0, 20);
} */
m_TestsModel.addElement(tname);
}
m_TestsModel.addElement("Summary");
m_TestsModel.addElement("Ranking");
// ================ BEGIN EDIT melville ================
m_TestsModel.addElement("Learning Curve Summary");
// ================ END EDIT melville ================
m_TestsList.setSelectedIndex(0);
m_TestsButton.setEnabled(true);
m_PerformBut.setEnabled(true);
}
public void setResultKeyFromDialog() {
ListSelectorDialog jd = new ListSelectorDialog(null, m_ResultKeyList);
// Open the dialog
int result = jd.showDialog();
// If accepted, update the ttester
if (result == ListSelectorDialog.APPROVE_OPTION) {
int[] selected = m_ResultKeyList.getSelectedIndices();
String selectedList = "";
for (int i = 0; i < selected.length; i++) {
selectedList += "," + (selected[i] + 1);
}
Range generatorRange = new Range();
if (selectedList.length() != 0) {
try {
generatorRange.setRanges(selectedList);
} catch (Exception ex) {
ex.printStackTrace();
System.err.println(ex.getMessage());
}
}
m_TTester.setResultsetKeyColumns(generatorRange);
setTTester();
}
}
/** Carries out a t-test using the current configuration. */
protected void performTest() {
String sigStr = m_SigTex.getText();
if (sigStr.length() != 0) {
m_TTester.setSignificanceLevel((new Double(sigStr)).doubleValue());
} else {
m_TTester.setSignificanceLevel(0.05);
}
String precStr = m_PrecTex.getText();
if (precStr.length() != 0) {
m_TTester.setPrecision((new Integer(precStr)).intValue());
} else {
m_TTester.setPrecision(2);
}
// Carry out the test chosen and biff the results to the output area
m_TTester.setShowStdDevs(m_ShowStdDevs.isSelected());
int compareCol = m_CompareCombo.getSelectedIndex();
int tType = m_TestsList.getSelectedIndex();
// =============== BEGIN EDIT melville ===============
String test_selected = (String) m_TestsList.getSelectedValue();
String name =
(new SimpleDateFormat("HH:mm:ss - ")).format(new Date())
+ (String) m_CompareCombo.getSelectedItem()
+ " - "
+ test_selected;
StringBuffer outBuff = new StringBuffer();
if (((String) m_CompareCombo.getSelectedItem()).equalsIgnoreCase("%error_reduction"))
outBuff.append(m_TTester.header("Percentage error reduction"));
else if (((String) m_CompareCombo.getSelectedItem())
.equalsIgnoreCase("top_20%_%error_reduction"))
outBuff.append(m_TTester.header("Top 20% Percentage error reduction"));
else outBuff.append(m_TTester.header(compareCol));
outBuff.append("\n");
m_History.addResult(name, outBuff);
m_History.setSingle(name);
try {
if (tType < m_TTester.getNumResultsets()) {
if (((String) m_CompareCombo.getSelectedItem()).equalsIgnoreCase("%error_reduction")) {
outBuff.append(m_TTester.multiResultsetPercentErrorReduction(tType, m_ErrorCompareCol));
} else if (((String) m_CompareCombo.getSelectedItem())
.equalsIgnoreCase("top_20%_%error_reduction")) {
outBuff.append(
m_TTester.multiResultsetTopNPercentErrorReduction(tType, m_ErrorCompareCol, 0.20));
} else outBuff.append(m_TTester.multiResultsetFull(tType, compareCol));
} else if (test_selected.equalsIgnoreCase("summary")) {
outBuff.append(m_TTester.multiResultsetSummary(compareCol));
} else if (test_selected.equalsIgnoreCase("ranking")) {
outBuff.append(m_TTester.multiResultsetRanking(compareCol));
}
// ================ END EDIT melville ================
outBuff.append("\n");
} catch (Exception ex) {
outBuff.append(ex.getMessage() + "\n");
}
m_History.updateResult(name);
}
/**
* Sets up the panel with a new set of instances, attempting to guess the correct settings for
* various columns.
*
* @param newInstances the new set of results.
*/
public void setInstances(Instances newInstances) {
m_Instances = newInstances;
m_TTester.setInstances(m_Instances);
m_FromLab.setText("Got " + m_Instances.numInstances() + " results");
// Temporarily remove the configuration listener
m_RunCombo.removeActionListener(m_ConfigureListener);
// Do other stuff
m_DatasetKeyModel.removeAllElements();
m_RunModel.removeAllElements();
m_ResultKeyModel.removeAllElements();
m_CompareModel.removeAllElements();
int datasetCol = -1;
int runCol = -1;
String selectedList = "";
String selectedListDataset = "";
// =============== BEGIN EDIT melville ===============
boolean noise = false; // keep track of whether noise levels eval is required
boolean learning = false; // keep track of whether learning curve eval is required
boolean fraction =
false; // keep track of whether fractions of datasets are provided for learning
// the key on which to base the learning curves (either total instances or fraction)
int learning_key = -1;
boolean classificationTask =
false; // used to determine if regression measures should be selected
// =============== END EDIT melville ===============
for (int i = 0; i < m_Instances.numAttributes(); i++) {
String name = m_Instances.attribute(i).name();
m_DatasetKeyModel.addElement(name);
m_RunModel.addElement(name);
m_ResultKeyModel.addElement(name);
m_CompareModel.addElement(name);
// =============== BEGIN EDIT melville ===============
// If learning curves were generated then treat each
// dataset + pt combination as a different dataset
if (name.toLowerCase().equals("key_noise_levels")) {
// noise overrides learning curves - but treat noise levels
// like pts on learning curve
learning_key = i;
learning = true;
noise = true;
// fraction = true;
} else if (name.toLowerCase().equals("key_fraction_instances") && !noise) {
// fraction overrides total_instances
learning_key = i;
learning = true;
fraction = true;
} else if (name.toLowerCase().equals("key_total_instances") && !learning) {
learning_key = i;
learning = true;
} else
// =============== END EDIT melville ===============
if (name.toLowerCase().equals("key_dataset")) {
m_DatasetKeyList.addSelectionInterval(i, i);
selectedListDataset += "," + (i + 1);
} else if ((runCol == -1) && (name.toLowerCase().equals("key_run"))) {
m_RunCombo.setSelectedIndex(i);
runCol = i;
} else if (name.toLowerCase().equals("key_scheme")
|| name.toLowerCase().equals("key_scheme_options")
|| name.toLowerCase().equals("key_scheme_version_id")) {
m_ResultKeyList.addSelectionInterval(i, i);
selectedList += "," + (i + 1);
// =============== BEGIN EDIT mbilenko ===============
// automatic selection of the correct measure for clustering experiments
} else if (name.toLowerCase().indexOf("pairwise_f_measure") != -1) {
m_CompareCombo.setSelectedIndex(i);
m_ErrorCompareCol = i;
}
// automatic selection of the correct measure for deduping experiments
else if (name.toLowerCase().equals("precision")) {
m_CompareCombo.setSelectedIndex(i);
// =============== END EDIT mbilenko ===============
} else if (name.toLowerCase().indexOf("percent_correct") != -1) {
m_CompareCombo.setSelectedIndex(i);
classificationTask = true;
} else if (!classificationTask
&& (name.toLowerCase().indexOf("root_mean_squared_error") != -1)) {
// automatic selection of the correct measure for regression experiments
m_CompareCombo.setSelectedIndex(i);
} else if (name.toLowerCase().indexOf("percent_incorrect") != -1) {
m_ErrorCompareCol = i;
// remember index of error for computing error reductions
}
}
// =============== BEGIN EDIT melville ===============
if (learning) {
m_DatasetKeyList.addSelectionInterval(learning_key, learning_key);
selectedListDataset += "," + (learning_key + 1);
m_CompareModel.addElement("%Error_reduction");
m_CompareModel.addElement("Top_20%_%Error_reduction");
}
// =============== END EDIT melville ===============
if (runCol == -1) {
runCol = 0;
}
m_DatasetKeyBut.setEnabled(true);
m_RunCombo.setEnabled(true);
m_ResultKeyBut.setEnabled(true);
m_CompareCombo.setEnabled(true);
// Reconnect the configuration listener
m_RunCombo.addActionListener(m_ConfigureListener);
// Set up the TTester with the new data
m_TTester.setRunColumn(runCol);
Range generatorRange = new Range();
if (selectedList.length() != 0) {
try {
generatorRange.setRanges(selectedList);
} catch (Exception ex) {
ex.printStackTrace();
System.err.println(ex.getMessage());
}
}
m_TTester.setResultsetKeyColumns(generatorRange);
generatorRange = new Range();
if (selectedListDataset.length() != 0) {
try {
generatorRange.setRanges(selectedListDataset);
} catch (Exception ex) {
ex.printStackTrace();
System.err.println(ex.getMessage());
}
}
m_TTester.setDatasetKeyColumns(generatorRange);
// =============== BEGIN EDIT melville ===============
m_TTester.setLearningCurve(learning);
m_TTester.setFraction(fraction);
if (learning) { // get points on the learning curve
Attribute attr;
if (noise) { // override fraction
attr = m_Instances.attribute("Key_Noise_levels");
} else if (fraction) {
attr = m_Instances.attribute("Key_Fraction_instances");
} else {
attr = m_Instances.attribute("Key_Total_instances");
}
double[] pts = new double[attr.numValues()];
for (int k = 0; k < attr.numValues(); k++) {
pts[k] = Double.parseDouble(attr.value(k));
}
// sort points
Arrays.sort(pts);
m_TTester.setPoints(pts);
}
// =============== END EDIT melville ===============
m_SigTex.setEnabled(true);
m_PrecTex.setEnabled(true);
setTTester();
}
public void actionPerformed(ActionEvent e) {
m_TTester.setRunColumn(m_RunCombo.getSelectedIndex());
setTTester();
}
public void setDatasetKeyFromDialog() {
ListSelectorDialog jd = new ListSelectorDialog(null, m_DatasetKeyList);
// Open the dialog
int result = jd.showDialog();
// =============== BEGIN EDIT melville ===============
// Check if learning curves should be generated
boolean noise = false;
boolean learning = false;
boolean fraction = false;
int learning_key = -1;
// If accepted, update the ttester
if (result == ListSelectorDialog.APPROVE_OPTION) {
int[] selected = m_DatasetKeyList.getSelectedIndices();
String selectedList = "";
Object[] selected_string = m_DatasetKeyList.getSelectedValues();
for (int i = 0; i < selected.length; i++) {
if (((String) selected_string[i]).toLowerCase().equals("key_noise_levels")) {
learning_key = i;
learning = true;
// fraction = true;
noise = true;
} else if (((String) selected_string[i]).toLowerCase().equals("key_fraction_instances")) {
learning_key = i;
learning = true;
fraction = true;
} else if (((String) selected_string[i]).toLowerCase().equals("key_total_instances")
&& !learning) {
learning = true;
learning_key = i;
} else selectedList += "," + (selected[i] + 1);
}
m_TTester.setLearningCurve(learning);
m_TTester.setFraction(fraction);
if (learning) { // get points on the learning curve
selectedList += "," + (selected[learning_key] + 1);
Attribute attr;
if (noise) { // override fraction
attr = m_Instances.attribute("Key_Noise_levels");
} else if (fraction) {
attr = m_Instances.attribute("Key_Fraction_instances");
} else {
attr = m_Instances.attribute("Key_Total_instances");
}
double[] pts = new double[attr.numValues()];
for (int k = 0; k < attr.numValues(); k++) {
pts[k] = Double.parseDouble(attr.value(k));
}
Arrays.sort(pts);
m_TTester.setPoints(pts);
}
// ================ END EDIT melville ================
Range generatorRange = new Range();
if (selectedList.length() != 0) {
try {
generatorRange.setRanges(selectedList);
} catch (Exception ex) {
ex.printStackTrace();
System.err.println(ex.getMessage());
}
}
m_TTester.setDatasetKeyColumns(generatorRange);
setTTester();
}
}