/**
* Runs a regression test -- this checks that the output of the tested object matches that in a
* reference version. When this test is run without any pre-existing reference output, the
* reference version is created.
*/
public void testRegression() throws Exception {
int i;
boolean succeeded;
Regression reg;
Instances train;
// don't bother if not working correctly
if (m_Tester.hasClasspathProblems()) return;
reg = new Regression(this.getClass());
succeeded = false;
train = null;
for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) {
// does the Kernel support this type of class at all?
if (!canPredict(i)) continue;
train =
m_Tester.makeTestDataset(
42,
m_Tester.getNumInstances(),
m_NominalPredictors[i] ? 2 : 0,
m_NumericPredictors[i] ? 1 : 0,
m_StringPredictors[i] ? 1 : 0,
m_DatePredictors[i] ? 1 : 0,
m_RelationalPredictors[i] ? 1 : 0,
2,
i,
m_multiInstanceHandler);
try {
m_RegressionResults[i] =
CheckKernel.attributeTypeToString(i) + " class:\n" + useKernel(train);
succeeded = true;
reg.println(m_RegressionResults[i]);
} catch (Exception e) {
String msg = e.getMessage().toLowerCase();
if (msg.indexOf("not in classpath") > -1) return;
m_RegressionResults[i] = null;
}
}
if (!succeeded) {
fail("Problem during regression testing: no successful predictions for any class type");
}
try {
String diff = reg.diff();
if (diff == null) {
System.err.println("Warning: No reference available, creating.");
} else if (!diff.equals("")) {
fail("Regression test failed. Difference:\n" + diff);
}
} catch (java.io.IOException ex) {
fail("Problem during regression testing.\n" + ex);
}
}
/**
* checks whether the Kernel can handle the given percentage of missing class labels
*
* @param type the class type
* @param percent the percentage of missing class labels
* @param allowFail if true a fail statement may be executed
* @return true if the Kernel can handle it
*/
protected boolean checkMissingClass(int type, int percent, boolean allowFail) {
boolean[] result;
result =
m_Tester.canHandleMissing(
m_NominalPredictors[type],
m_NumericPredictors[type],
m_StringPredictors[type],
m_DatePredictors[type],
m_RelationalPredictors[type],
m_multiInstanceHandler,
type,
false,
true,
percent);
if (allowFail) {
if (!result[0] && !result[1])
fail(
"Error handling "
+ percent
+ "% missing class labels ("
+ getClassTypeString(type)
+ " class)!");
}
return result[0];
}
/**
* checks whether the Kernel can handle the class attribute at a given position (0-based index, -1
* means last).
*
* @param type the class type
* @param position the position of the class attribute (0-based, -1 means last)
* @return true if the Kernel can handle it
*/
protected boolean checkClassAsNthAttribute(int type, int position) {
boolean[] result;
String indexStr;
result =
m_Tester.canHandleClassAsNthAttribute(
m_NominalPredictors[type],
m_NumericPredictors[type],
m_StringPredictors[type],
m_DatePredictors[type],
m_RelationalPredictors[type],
m_multiInstanceHandler,
type,
position);
if (position == -1) indexStr = "last";
else indexStr = (position + 1) + ".";
if (!result[0] && !result[1])
fail(
"Error handling class as "
+ indexStr
+ " attribute ("
+ getClassTypeString(type)
+ " class)!");
return result[0];
}
/**
* tests whether the Kernel handles instance weights correctly
*
* @see CheckKernel#instanceWeights(boolean, boolean, boolean, boolean, boolean, boolean, int)
* @see CheckKernel#testsPerClassType(int, boolean, boolean)
*/
public void testInstanceWeights() {
boolean[] result;
int i;
if (m_weightedInstancesHandler) {
for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) {
// does the Kernel support this type of class at all?
if (!canPredict(i)) continue;
result =
m_Tester.instanceWeights(
m_NominalPredictors[i],
m_NumericPredictors[i],
m_StringPredictors[i],
m_DatePredictors[i],
m_RelationalPredictors[i],
m_multiInstanceHandler,
i);
if (!result[0])
System.err.println(
"Error handling instance weights (" + getClassTypeString(i) + " class)!");
}
}
}
/** tests whether the scheme declares a serialVersionUID. */
public void testSerialVersionUID() {
boolean[] result;
result = m_Tester.declaresSerialVersionUID();
if (!result[0]) fail("Doesn't declare serialVersionUID!");
}
/**
* tests whether the Kernel can handle certain attributes and if not, if the exception is OK
*
* @param nom to check for nominal attributes
* @param num to check for numeric attributes
* @param str to check for string attributes
* @param dat to check for date attributes
* @param rel to check for relational attributes
* @param allowFail whether a junit fail can be executed
* @see CheckKernel#canPredict(boolean, boolean, boolean, boolean, boolean, boolean, int)
* @see CheckKernel#testsPerClassType(int, boolean, boolean)
*/
protected void checkAttributes(
boolean nom, boolean num, boolean str, boolean dat, boolean rel, boolean allowFail) {
boolean[] result;
String att;
int i;
// determine text for type of attributes
att = "";
if (nom) att = "nominal";
else if (num) att = "numeric";
else if (str) att = "string";
else if (dat) att = "date";
else if (rel) att = "relational";
for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) {
result = m_Tester.canPredict(nom, num, str, dat, rel, m_multiInstanceHandler, i);
if (nom) m_NominalPredictors[i] = result[0];
else if (num) m_NumericPredictors[i] = result[0];
else if (str) m_StringPredictors[i] = result[0];
else if (dat) m_DatePredictors[i] = result[0];
else if (rel) m_RelationalPredictors[i] = result[0];
if (!result[0] && !result[1] && allowFail)
fail("Error handling " + att + " attributes (" + getClassTypeString(i) + " class)!");
}
}
/**
* Called by JUnit before each test method. This implementation creates the default Kernel to test
* and loads a test set of Instances.
*
* @exception Exception if an error occurs reading the example instances.
*/
protected void setUp() throws Exception {
m_Kernel = getKernel();
m_Tester = getTester();
m_OptionTester = getOptionTester();
m_GOETester = getGOETester();
m_weightedInstancesHandler = m_Tester.weightedInstancesHandler()[0];
m_multiInstanceHandler = m_Tester.multiInstanceHandler()[0];
m_NominalPredictors = new boolean[LAST_CLASSTYPE + 1];
m_NumericPredictors = new boolean[LAST_CLASSTYPE + 1];
m_StringPredictors = new boolean[LAST_CLASSTYPE + 1];
m_DatePredictors = new boolean[LAST_CLASSTYPE + 1];
m_RelationalPredictors = new boolean[LAST_CLASSTYPE + 1];
m_handleMissingPredictors = new boolean[LAST_CLASSTYPE + 1];
m_handleMissingClass = new boolean[LAST_CLASSTYPE + 1];
m_handleClassAsFirstAttribute = new boolean[LAST_CLASSTYPE + 1];
m_handleClassAsSecondAttribute = new boolean[LAST_CLASSTYPE + 1];
m_RegressionResults = new String[LAST_CLASSTYPE + 1];
m_NClasses = 4;
// initialize attributes
checkAttributes(true, false, false, false, false, false);
checkAttributes(false, true, false, false, false, false);
checkAttributes(false, false, true, false, false, false);
checkAttributes(false, false, false, true, false, false);
checkAttributes(false, false, false, false, true, false);
// initialize missing values handling
for (int i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) {
// does the scheme support this type of class at all?
if (!canPredict(i)) continue;
// 20% missing
m_handleMissingPredictors[i] = checkMissingPredictors(i, 20, false);
m_handleMissingClass[i] = checkMissingClass(i, 20, false);
}
}
/**
* tests whether Kernel handles N classes
*
* @see CheckKernel#canHandleNClasses(boolean, boolean, boolean, boolean, boolean, boolean, int)
* @see CheckKernel#testsPerClassType(int, boolean, boolean)
* @see #m_NClasses
*/
public void testNClasses() {
boolean[] result;
if (!canPredict(Attribute.NOMINAL)) return;
result =
m_Tester.canHandleNClasses(
m_NominalPredictors[Attribute.NOMINAL],
m_NumericPredictors[Attribute.NOMINAL],
m_StringPredictors[Attribute.NOMINAL],
m_DatePredictors[Attribute.NOMINAL],
m_RelationalPredictors[Attribute.NOMINAL],
m_multiInstanceHandler,
m_NClasses);
if (!result[0] && !result[1]) fail("Error handling " + m_NClasses + " classes!");
}
/**
* configures the CheckKernel instance used throughout the tests
*
* @return the fully configured CheckKernel instance used for testing
*/
protected CheckKernel getTester() {
CheckKernel result;
result = new CheckKernel();
result.setSilent(true);
result.setKernel(m_Kernel);
result.setNumInstances(20);
result.setDebug(DEBUG);
result.setPostProcessor(getPostProcessor());
return result;
}
/**
* tests whether the Kernel correctly initializes in the buildKernel method
*
* @see CheckKernel#correctBuildInitialisation(boolean, boolean, boolean, boolean, boolean,
* boolean, int)
* @see CheckKernel#testsPerClassType(int, boolean, boolean)
*/
public void testBuildInitialization() {
boolean[] result;
int i;
for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) {
// does the Kernel support this type of class at all?
if (!canPredict(i)) continue;
result =
m_Tester.correctBuildInitialisation(
m_NominalPredictors[i],
m_NumericPredictors[i],
m_StringPredictors[i],
m_DatePredictors[i],
m_RelationalPredictors[i],
m_multiInstanceHandler,
i);
if (!result[0] && !result[1])
fail("Incorrect build initialization (" + getClassTypeString(i) + " class)!");
}
}
/**
* tests whether the Kernel can handle zero training instances
*
* @see CheckKernel#canHandleZeroTraining(boolean, boolean, boolean, boolean, boolean, boolean,
* int)
* @see CheckKernel#testsPerClassType(int, boolean, boolean)
*/
public void testZeroTraining() {
boolean[] result;
int i;
for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) {
// does the Kernel support this type of class at all?
if (!canPredict(i)) continue;
result =
m_Tester.canHandleZeroTraining(
m_NominalPredictors[i],
m_NumericPredictors[i],
m_StringPredictors[i],
m_DatePredictors[i],
m_RelationalPredictors[i],
m_multiInstanceHandler,
i);
if (!result[0] && !result[1])
fail("Error handling zero training instances (" + getClassTypeString(i) + " class)!");
}
}
/**
* tests whether the Kernel alters the training set during training.
*
* @see CheckKernel#datasetIntegrity(boolean, boolean, boolean, boolean, boolean, boolean, int,
* boolean, boolean)
* @see CheckKernel#testsPerClassType(int, boolean, boolean)
*/
public void testDatasetIntegrity() {
boolean[] result;
int i;
for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) {
// does the Kernel support this type of class at all?
if (!canPredict(i)) continue;
result =
m_Tester.datasetIntegrity(
m_NominalPredictors[i],
m_NumericPredictors[i],
m_StringPredictors[i],
m_DatePredictors[i],
m_RelationalPredictors[i],
m_multiInstanceHandler,
i,
m_handleMissingPredictors[i],
m_handleMissingClass[i]);
if (!result[0] && !result[1])
fail("Training set is altered during training (" + getClassTypeString(i) + " class)!");
}
}
/**
* returns a string for the class type
*
* @param type the class type
* @return the class type as string
*/
protected String getClassTypeString(int type) {
return CheckKernel.attributeTypeToString(type);
}
