@Override
public void iterationDone(int epoch) {
int plotEpochs = getRenderIterations();
if (plotEpochs <= 0) return;
if (epoch % plotEpochs == 0 || epoch == 0) {
NeuralNetPlotter plotter = new NeuralNetPlotter();
plotter.plotNetworkGradient(
this, this.getGradient(new Object[] {1, 0.001, 1000}), getInput().rows);
}
}
public static void main(String[] args) throws Exception {
// batches of 10, 60000 examples total
DataSetIterator iter = new MnistDataSetIterator(10, 300);
DBN dbn = SerializationUtils.readObject(new File(args[0]));
for (int i = 0; i < dbn.getnLayers(); i++) dbn.getLayers()[i].setRenderEpochs(10);
DeepAutoEncoder encoder = new DeepAutoEncoder(dbn);
encoder.setRoundCodeLayerInput(true);
encoder.setNormalizeCodeLayerOutput(false);
encoder.setOutputLayerLossFunction(OutputLayer.LossFunction.RMSE_XENT);
encoder.setOutputLayerActivation(Activations.sigmoid());
encoder.setVisibleUnit(RBM.VisibleUnit.GAUSSIAN);
encoder.setHiddenUnit(RBM.HiddenUnit.BINARY);
while (iter.hasNext()) {
DataSet next = iter.next();
if (next == null) break;
log.info("Training on " + next.numExamples());
log.info("Coding layer is " + encoder.encode(next.getFirst()));
encoder.finetune(next.getFirst(), 1e-1, 1000);
NeuralNetPlotter plotter = new NeuralNetPlotter();
encoder.getDecoder().feedForward(encoder.encodeWithScaling(next.getFirst()));
String[] layers = new String[encoder.getDecoder().getnLayers()];
DoubleMatrix[] weights = new DoubleMatrix[layers.length];
for (int i = 0; i < encoder.getDecoder().getnLayers(); i++) {
layers[i] = "" + i;
weights[i] = encoder.getDecoder().getLayers()[i].getW();
}
plotter.histogram(layers, weights);
FilterRenderer f = new FilterRenderer();
f.renderFilters(
encoder.getDecoder().getOutputLayer().getW(),
"outputlayer.png",
28,
28,
next.numExamples());
DeepAutoEncoderDataSetReconstructionRender render =
new DeepAutoEncoderDataSetReconstructionRender(
next.iterator(next.numExamples()), encoder);
render.draw();
}
SerializationUtils.saveObject(encoder, new File("deepautoencoder.ser"));
iter.reset();
DeepAutoEncoderDataSetReconstructionRender render =
new DeepAutoEncoderDataSetReconstructionRender(iter, encoder);
render.draw();
}
@Override
public void iterationDone(int iterationDone) {
int plotEpochs = network.conf().getRenderWeightsEveryNumEpochs();
if (plotEpochs <= 0) return;
if (iterationDone % plotEpochs == 0) {
plotter.plotNetworkGradient(network, network.getGradient(extraParams), 100);
}
}
/** Backprop with the output being the reconstruction */
@Override
public void backProp(double lr, int iterations, Object[] extraParams) {
double currRecon = squaredLoss();
boolean train = true;
NeuralNetwork revert = clone();
while (train) {
if (iterations > iterations) break;
double newRecon = this.squaredLoss();
// prevent weights from exploding too far in either direction, we want this as close to zero
// as possible
if (newRecon > currRecon || currRecon < 0 && newRecon < currRecon) {
update((BaseNeuralNetwork) revert);
log.info("Converged for new recon; breaking...");
break;
} else if (Double.isNaN(newRecon) || Double.isInfinite(newRecon)) {
update((BaseNeuralNetwork) revert);
log.info("Converged for new recon; breaking...");
break;
} else if (newRecon == currRecon) break;
else {
currRecon = newRecon;
revert = clone();
log.info("Recon went down " + currRecon);
}
iterations++;
int plotIterations = getRenderIterations();
if (plotIterations > 0) {
NeuralNetPlotter plotter = new NeuralNetPlotter();
if (iterations % plotIterations == 0) {
plotter.plotNetworkGradient(this, getGradient(extraParams), getInput().rows);
}
}
}
}
public static void main(String... args) throws Exception {
int numFeatures = 40;
int iterations = 5;
int seed = 123;
int listenerFreq = iterations / 5;
Nd4j.getRandom().setSeed(seed);
log.info("Load dat....");
INDArray input =
Nd4j.create(
2,
numFeatures); // have to be at least two or else output layer gradient is a scalar and
// cause exception
INDArray labels = Nd4j.create(2, 2);
INDArray row0 = Nd4j.create(1, numFeatures);
row0.assign(0.1);
input.putRow(0, row0);
labels.put(0, 1, 1); // set the 4th column
INDArray row1 = Nd4j.create(1, numFeatures);
row1.assign(0.2);
input.putRow(1, row1);
labels.put(1, 0, 1); // set the 2nd column
DataSet trainingSet = new DataSet(input, labels);
log.info("Build model....");
NeuralNetConfiguration conf =
new NeuralNetConfiguration.Builder()
.layer(new RBM())
.nIn(trainingSet.numInputs())
.nOut(trainingSet.numOutcomes())
.seed(seed)
.weightInit(WeightInit.SIZE)
.constrainGradientToUnitNorm(true)
.iterations(iterations)
.activationFunction("tanh")
.visibleUnit(RBM.VisibleUnit.GAUSSIAN)
.hiddenUnit(RBM.HiddenUnit.RECTIFIED)
.lossFunction(LossFunctions.LossFunction.RMSE_XENT)
.learningRate(1e-1f)
.optimizationAlgo(OptimizationAlgorithm.ITERATION_GRADIENT_DESCENT)
.build();
Layer model = LayerFactories.getFactory(conf).create(conf);
model.setIterationListeners(
Collections.singletonList((IterationListener) new ScoreIterationListener(listenerFreq)));
log.info("Evaluate weights....");
INDArray w = model.getParam(DefaultParamInitializer.WEIGHT_KEY);
log.info("Weights: " + w);
log.info("Train model....");
model.fit(trainingSet.getFeatureMatrix());
log.info("Visualize training results....");
// Work in progress to get NeuralNetPlotter functioning
NeuralNetPlotter plotter = new NeuralNetPlotter();
plotter.plotNetworkGradient(model, model.gradient(), 10);
}
