/**
* Run an experiment, iterating through all possible combinations of parameters.
*
* @param sizes
* @param dimensions
* @param repeats
* @param rbfBasisFunction
* @param activationFunction
* @param learningRate
* @param clusters {number of hidden nodes in each layer, number of clusters}
* @param momentum
* @param hiddenNum number of hidden layers
*/
public static void runExperiment(
int[] sizes,
int[] dimensions,
int[][] repeats,
int[] rbfBasisFunction,
ActivationFunction[] activationFunction,
double[] learningRate,
int[][] clusters,
double[] momentum,
int[] hiddenNum) {
// Check for empty and null-valued arguments and initialize to default values.
if (sizes == null || sizes.length == 0) {
sizes = new int[] {100000};
}
if (dimensions == null || dimensions.length == 0) {
dimensions = new int[] {4};
}
if (repeats == null || repeats.length == 0) {
repeats = new int[][] {{1, 1}};
}
if (rbfBasisFunction == null || rbfBasisFunction.length == 0) {
rbfBasisFunction = new int[] {0};
}
if (activationFunction == null || activationFunction.length == 0) {
activationFunction = new ActivationFunction[] {ActivationFunction.LOGISTIC};
}
if (learningRate == null || learningRate.length == 0) {
learningRate = new double[] {0.01};
}
if (clusters == null || clusters.length == 0) {
clusters = new int[][] {{100, 10000}};
}
if (momentum == null || momentum.length == 0) {
momentum = new double[] {0.01};
}
if (hiddenNum == null || hiddenNum.length == 0) {
hiddenNum = new int[] {1};
}
// calculate the number of experiments
int experimentCount =
sizes.length
* repeats.length
* dimensions.length
* rbfBasisFunction.length
* activationFunction.length
* learningRate.length
* clusters.length
* momentum.length
* hiddenNum.length;
double[][][][] results = new double[2][experimentCount][][];
int experimentIndex = 0;
// Loop through all possible combinations of variables (most arrays should be of size 1)
for (int a = 0; a < sizes.length; a++) {
for (int b = 0; b < dimensions.length; b++) {
// Retrieve a dataset of size a with b inputs, or generate one if it does not exist
double[][] datasets = DataTools.getDataFromFile(dimensions[b], sizes[a]);
for (int c = 0; c < repeats.length; c++) {
for (int d = 0; d < rbfBasisFunction.length; d++) {
for (int e = 0; e < activationFunction.length; e++) {
for (int f = 0; f < learningRate.length; f++) {
for (int g = 0; g < clusters.length; g++) {
for (int h = 0; h < momentum.length; h++) {
for (int i = 0; i < hiddenNum.length; i++) {
// Print information about the next experiment
System.out.println("Experiment " + experimentIndex);
System.out.println("Number of training examples: " + sizes[a]);
System.out.println("Number of inputs: " + dimensions[b]);
System.out.println("Repeats: {" + repeats[c][0] + "," + repeats[c][1] + "}");
System.out.println("RBF basis function: " + rbfBasisFunction[d]);
System.out.println("FF activation function: " + activationFunction[e]);
System.out.println("Learning rate: " + learningRate[f]);
System.out.println("Clusters: " + clusters[g][1]);
System.out.println("Momentum: " + momentum[h]);
System.out.println("Hidden layers: " + hiddenNum[i]);
System.out.println();
// Train and test an RBF neural net
results[0][experimentIndex] =
trainRBF(
sizes[a],
dimensions[b],
repeats[c],
rbfBasisFunction[d],
activationFunction[e],
learningRate[f],
clusters[g],
momentum[h],
hiddenNum[i],
datasets);
double aboveRight = results[0][experimentIndex][0][0];
double aboveWrong = results[0][experimentIndex][0][1];
double belowRight = results[0][experimentIndex][0][2];
double belowWrong = results[0][experimentIndex][0][3];
double varianceSum = results[0][experimentIndex][0][4];
double realSum = results[0][experimentIndex][0][5];
double predictionSum = results[0][experimentIndex][0][6];
double minError = results[0][experimentIndex][0][7];
double maxError = results[0][experimentIndex][0][8];
double percentError = results[0][experimentIndex][0][9] / (sizes[a] * 10);
double averageError = varianceSum / (sizes[a] * 10);
double standardDeviation = 0.0;
double averageReal = realSum / (sizes[a] * 10);
double averagePrediction = predictionSum / (sizes[a] * 10);
double[] errors = results[0][experimentIndex][1];
for (int j = 0; j < errors.length; j++) {
double error = results[0][experimentIndex][1][j];
standardDeviation += Math.pow(error - averageError, 2.0);
}
standardDeviation /= 10 * sizes[a] - 1;
standardDeviation = Math.sqrt(standardDeviation);
System.out.println("------------RBF Neural Network------------");
System.out.println(" Correct above guesses: " + aboveRight);
System.out.println(" Incorrect above guesses: " + aboveWrong);
System.out.println(" Correct below guesses: " + belowRight);
System.out.println(" Incorrect below guesses: " + belowWrong);
System.out.println(" Average real value: " + averageReal);
System.out.println(" Average predicted value: " + averagePrediction);
System.out.println(" Average error: " + averageError);
System.out.println(" Average percent error: " + percentError);
System.out.println(" Standard dev. of errors: " + standardDeviation);
System.out.println(" Minimum error: " + minError);
System.out.println(" Maximum error: " + maxError);
System.out.println(
"Percent correctly guessed: "
+ ((aboveRight + belowRight)
/ (aboveRight + belowRight + aboveWrong + belowWrong)
* 100));
System.out.println();
// Train and test a Feedforward neural net
results[1][experimentIndex] =
trainFF(
sizes[a],
dimensions[b],
repeats[c],
rbfBasisFunction[d],
activationFunction[e],
learningRate[f],
clusters[g],
momentum[h],
hiddenNum[i],
datasets);
double[] statistics = new double[5];
for (int j = 0; j < 5; j++) {
for (int k = 0; k < 10; k++) {
statistics[k % 5] += results[1][experimentIndex][j][k];
}
}
statistics[4] /= 10;
double[] confidences = results[1][experimentIndex][6];
confidences[0] /= statistics[0];
confidences[1] /= statistics[1];
confidences[2] /= statistics[2];
confidences[3] /= statistics[3];
System.out.println("--------Feed Forward Neural Network--------");
System.out.println(" Correct above guesses: " + statistics[0]);
System.out.println("Incorrect above guesses: " + statistics[1]);
System.out.println(" Correct below guesses: " + statistics[2]);
System.out.println("Incorrect below guesses: " + statistics[3]);
System.out.println(" Average confidence: " + statistics[4]);
System.out.println("Average confidence when...");
System.out.println(" Correct and above: " + confidences[0]);
System.out.println(" Incorrect and above: " + confidences[1]);
System.out.println(" Correct and below: " + confidences[2]);
System.out.println(" Incorrect and below: " + confidences[3]);
System.out.println();
experimentIndex++;
}
}
}
}
}
}
}
}
}
}
