public static void main(String[] args) throws Exception {
// batches of 10, 60000 examples total
DataSetIterator iter = new MnistDataSetIterator(10, 300);
DBN dbn = SerializationUtils.readObject(new File(args[0]));
for (int i = 0; i < dbn.getnLayers(); i++) dbn.getLayers()[i].setRenderEpochs(10);
DeepAutoEncoder encoder = new DeepAutoEncoder(dbn);
encoder.setRoundCodeLayerInput(true);
encoder.setNormalizeCodeLayerOutput(false);
encoder.setOutputLayerLossFunction(OutputLayer.LossFunction.RMSE_XENT);
encoder.setOutputLayerActivation(Activations.sigmoid());
encoder.setVisibleUnit(RBM.VisibleUnit.GAUSSIAN);
encoder.setHiddenUnit(RBM.HiddenUnit.BINARY);
while (iter.hasNext()) {
DataSet next = iter.next();
if (next == null) break;
log.info("Training on " + next.numExamples());
log.info("Coding layer is " + encoder.encode(next.getFirst()));
encoder.finetune(next.getFirst(), 1e-1, 1000);
NeuralNetPlotter plotter = new NeuralNetPlotter();
encoder.getDecoder().feedForward(encoder.encodeWithScaling(next.getFirst()));
String[] layers = new String[encoder.getDecoder().getnLayers()];
DoubleMatrix[] weights = new DoubleMatrix[layers.length];
for (int i = 0; i < encoder.getDecoder().getnLayers(); i++) {
layers[i] = "" + i;
weights[i] = encoder.getDecoder().getLayers()[i].getW();
}
plotter.histogram(layers, weights);
FilterRenderer f = new FilterRenderer();
f.renderFilters(
encoder.getDecoder().getOutputLayer().getW(),
"outputlayer.png",
28,
28,
next.numExamples());
DeepAutoEncoderDataSetReconstructionRender render =
new DeepAutoEncoderDataSetReconstructionRender(
next.iterator(next.numExamples()), encoder);
render.draw();
}
SerializationUtils.saveObject(encoder, new File("deepautoencoder.ser"));
iter.reset();
DeepAutoEncoderDataSetReconstructionRender render =
new DeepAutoEncoderDataSetReconstructionRender(iter, encoder);
render.draw();
}
public static void main(String[] args) throws Exception {
final int numRows = 28;
final int numColumns = 28;
int outputNum = 10;
int numSamples = 60000;
int batchSize = 100;
int iterations = 10;
int seed = 123;
int listenerFreq = batchSize / 5;
log.info("Load data....");
DataSetIterator iter = new MnistDataSetIterator(batchSize, numSamples, true);
log.info("Build model....");
MultiLayerNetwork model = softMaxRegression(seed, iterations, numRows, numColumns, outputNum);
// // MultiLayerNetwork model = deepBeliefNetwork(seed, iterations,
// // numRows, numColumns, outputNum);
// MultiLayerNetwork model = deepConvNetwork(seed, iterations, numRows,
// numColumns, outputNum);
model.init();
model.setListeners(
Collections.singletonList((IterationListener) new ScoreIterationListener(listenerFreq)));
log.info("Train model....");
model.fit(iter); // achieves end to end pre-training
log.info("Evaluate model....");
Evaluation eval = new Evaluation(outputNum);
DataSetIterator testIter = new MnistDataSetIterator(100, 10000);
while (testIter.hasNext()) {
DataSet testMnist = testIter.next();
INDArray predict2 = model.output(testMnist.getFeatureMatrix());
eval.eval(testMnist.getLabels(), predict2);
}
log.info(eval.stats());
log.info("****************Example finished********************");
}
public static void main(String[] args) throws Exception {
// Set up network. 784 in/out (as MNIST images are 28x28).
// 784 -> 250 -> 10 -> 250 -> 784
MultiLayerConfiguration conf =
new NeuralNetConfiguration.Builder()
.seed(12345)
.iterations(1)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.learningRate(0.05)
.l2(0.001)
.list(4)
.layer(
0,
new DenseLayer.Builder()
.nIn(784)
.nOut(250)
.weightInit(WeightInit.XAVIER)
.updater(Updater.ADAGRAD)
.activation("relu")
.build())
.layer(
1,
new DenseLayer.Builder()
.nIn(250)
.nOut(10)
.weightInit(WeightInit.XAVIER)
.updater(Updater.ADAGRAD)
.activation("relu")
.build())
.layer(
2,
new DenseLayer.Builder()
.nIn(10)
.nOut(250)
.weightInit(WeightInit.XAVIER)
.updater(Updater.ADAGRAD)
.activation("relu")
.build())
.layer(
3,
new OutputLayer.Builder()
.nIn(250)
.nOut(784)
.weightInit(WeightInit.XAVIER)
.updater(Updater.ADAGRAD)
.activation("relu")
.lossFunction(LossFunctions.LossFunction.MSE)
.build())
.pretrain(false)
.backprop(true)
.build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.setListeners(Arrays.asList((IterationListener) new ScoreIterationListener(1)));
// Load data and split into training and testing sets. 40000 train, 10000 test
DataSetIterator iter = new MnistDataSetIterator(100, 50000, false);
List<INDArray> featuresTrain = new ArrayList<>();
List<INDArray> featuresTest = new ArrayList<>();
List<INDArray> labelsTest = new ArrayList<>();
Random r = new Random(12345);
while (iter.hasNext()) {
DataSet ds = iter.next();
SplitTestAndTrain split = ds.splitTestAndTrain(80, r); // 80/20 split (from miniBatch = 100)
featuresTrain.add(split.getTrain().getFeatureMatrix());
DataSet dsTest = split.getTest();
featuresTest.add(dsTest.getFeatureMatrix());
INDArray indexes =
Nd4j.argMax(dsTest.getLabels(), 1); // Convert from one-hot representation -> index
labelsTest.add(indexes);
}
// Train model:
int nEpochs = 30;
for (int epoch = 0; epoch < nEpochs; epoch++) {
for (INDArray data : featuresTrain) {
net.fit(data, data);
}
System.out.println("Epoch " + epoch + " complete");
}
// Evaluate the model on test data
// Score each digit/example in test set separately
// Then add triple (score, digit, and INDArray data) to lists and sort by score
// This allows us to get best N and worst N digits for each type
Map<Integer, List<Triple<Double, Integer, INDArray>>> listsByDigit = new HashMap<>();
for (int i = 0; i < 10; i++)
listsByDigit.put(i, new ArrayList<Triple<Double, Integer, INDArray>>());
int count = 0;
for (int i = 0; i < featuresTest.size(); i++) {
INDArray testData = featuresTest.get(i);
INDArray labels = labelsTest.get(i);
int nRows = testData.rows();
for (int j = 0; j < nRows; j++) {
INDArray example = testData.getRow(j);
int label = (int) labels.getDouble(j);
double score = net.score(new DataSet(example, example));
listsByDigit.get(label).add(new ImmutableTriple<>(score, count++, example));
}
}
// Sort data by score, separately for each digit
Comparator<Triple<Double, Integer, INDArray>> c =
new Comparator<Triple<Double, Integer, INDArray>>() {
@Override
public int compare(
Triple<Double, Integer, INDArray> o1, Triple<Double, Integer, INDArray> o2) {
return Double.compare(o1.getLeft(), o2.getLeft());
}
};
for (List<Triple<Double, Integer, INDArray>> list : listsByDigit.values()) {
Collections.sort(list, c);
}
// Select the 5 best and 5 worst numbers (by reconstruction error) for each digit
List<INDArray> best = new ArrayList<>(50);
List<INDArray> worst = new ArrayList<>(50);
for (int i = 0; i < 10; i++) {
List<Triple<Double, Integer, INDArray>> list = listsByDigit.get(i);
for (int j = 0; j < 5; j++) {
best.add(list.get(j).getRight());
worst.add(list.get(list.size() - j - 1).getRight());
}
}
// Visualize the best and worst digits
MNISTVisualizer bestVisualizer = new MNISTVisualizer(2.0, best, "Best (Low Rec. Error)");
bestVisualizer.visualize();
MNISTVisualizer worstVisualizer = new MNISTVisualizer(2.0, worst, "Worst (High Rec. Error)");
worstVisualizer.visualize();
}
public void trainMLP() throws Exception {
Nd4j.ENFORCE_NUMERICAL_STABILITY = true;
final int numRows = 28;
final int numColumns = 28;
int outputNum = 10;
int numSamples = 10000;
int batchSize = 500;
int iterations = 10;
int seed = 123;
int listenerFreq = iterations / 5;
int splitTrainNum = (int) (batchSize * .8);
DataSet mnist;
SplitTestAndTrain trainTest;
DataSet trainInput;
List<INDArray> testInput = new ArrayList<>();
List<INDArray> testLabels = new ArrayList<>();
log.info("Load data....");
DataSetIterator mnistIter = new MnistDataSetIterator(batchSize, numSamples, true);
log.info("Build model....");
MultiLayerConfiguration conf =
new NeuralNetConfiguration.Builder()
.seed(seed)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.iterations(iterations)
.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
.learningRate(1e-1f)
.momentum(0.5)
.momentumAfter(Collections.singletonMap(3, 0.9))
.useDropConnect(true)
.list(2)
.layer(
0,
new DenseLayer.Builder()
.nIn(numRows * numColumns)
.nOut(1000)
.activation("relu")
.weightInit(WeightInit.XAVIER)
.build())
.layer(
1,
new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD)
.nIn(1000)
.nOut(outputNum)
.activation("softmax")
.weightInit(WeightInit.XAVIER)
.build())
.build();
MultiLayerNetwork model = new MultiLayerNetwork(conf);
model.init();
model.setListeners(Arrays.asList((IterationListener) new ScoreIterationListener(listenerFreq)));
log.info("Train model....");
model.setListeners(Arrays.asList((IterationListener) new ScoreIterationListener(listenerFreq)));
while (mnistIter.hasNext()) {
mnist = mnistIter.next();
trainTest =
mnist.splitTestAndTrain(splitTrainNum, new Random(seed)); // train set that is the result
trainInput = trainTest.getTrain(); // get feature matrix and labels for training
testInput.add(trainTest.getTest().getFeatureMatrix());
testLabels.add(trainTest.getTest().getLabels());
model.fit(trainInput);
}
log.info("Evaluate model....");
Evaluation eval = new Evaluation(outputNum);
for (int i = 0; i < testInput.size(); i++) {
INDArray output = model.output(testInput.get(i));
eval.eval(testLabels.get(i), output);
}
log.info(eval.stats());
log.info("****************Example finished********************");
}