@Test
  public void testDbnFaces() {
    DataSetIterator iter = new LFWDataSetIterator(28, 28);

    DataSet next = iter.next();
    next.normalizeZeroMeanZeroUnitVariance();

    MultiLayerConfiguration conf =
        new NeuralNetConfiguration.Builder()
            .nIn(next.numInputs())
            .nOut(next.numOutcomes())
            .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT)
            .constrainGradientToUnitNorm(true)
            .weightInit(WeightInit.DISTRIBUTION)
            .dist(new NormalDistribution(0, 1e-5))
            .iterations(10)
            .learningRate(1e-3)
            .lossFunction(LossFunctions.LossFunction.RMSE_XENT)
            .visibleUnit(RBM.VisibleUnit.GAUSSIAN)
            .hiddenUnit(RBM.HiddenUnit.RECTIFIED)
            .layer(new RBM())
            .list(4)
            .hiddenLayerSizes(600, 250, 100)
            .override(3, new ClassifierOverride())
            .build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    network.setListeners(
        Arrays.<IterationListener>asList(
            new ScoreIterationListener(10), new NeuralNetPlotterIterationListener(1)));
    network.fit(next);
  }
  @Test
  public void testGradientWithAsList() {
    MultiLayerNetwork net1 = new MultiLayerNetwork(getConf());
    MultiLayerNetwork net2 = new MultiLayerNetwork(getConf());
    net1.init();
    net2.init();

    DataSet x1 = new IrisDataSetIterator(1, 150).next();
    DataSet all = new IrisDataSetIterator(150, 150).next();
    DataSet x2 = all.asList().get(0);

    // x1 and x2 contain identical data
    assertArrayEquals(asFloat(x1.getFeatureMatrix()), asFloat(x2.getFeatureMatrix()), 0.0f); // OK
    assertArrayEquals(asFloat(x1.getLabels()), asFloat(x2.getLabels()), 0.0f); // OK
    assertEquals(x1, x2); // Fails, DataSet doesn't override Object.equals()

    // Set inputs/outputs so gradient can be calculated:
    net1.feedForward(x1.getFeatureMatrix());
    net2.feedForward(x2.getFeatureMatrix());
    ((OutputLayer) net1.getLayers()[1]).setLabels(x1.getLabels());
    ((OutputLayer) net2.getLayers()[1]).setLabels(x2.getLabels());

    net1.gradient(); // OK
    net2.gradient(); // IllegalArgumentException: Buffers must fill up specified length 29
  }
  @Test
  public void testGravesLSTMInit() {
    int nIn = 8;
    int nOut = 25;
    int nHiddenUnits = 17;
    MultiLayerConfiguration conf =
        new NeuralNetConfiguration.Builder()
            .list(2)
            .layer(
                0,
                new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder()
                    .nIn(nIn)
                    .nOut(nHiddenUnits)
                    .weightInit(WeightInit.DISTRIBUTION)
                    .activation("tanh")
                    .build())
            .layer(
                1,
                new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS)
                    .nIn(nHiddenUnits)
                    .nOut(nOut)
                    .weightInit(WeightInit.DISTRIBUTION)
                    .activation("tanh")
                    .build())
            .build();
    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    // Ensure that we have the correct number weights and biases, that these have correct shape etc.
    Layer layer = network.getLayer(0);
    assertTrue(layer instanceof GravesLSTM);

    Map<String, INDArray> paramTable = layer.paramTable();
    assertTrue(paramTable.size() == 3); // 2 sets of weights, 1 set of biases

    INDArray recurrentWeights = paramTable.get(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY);
    assertArrayEquals(
        recurrentWeights.shape(),
        new int[] {
          nHiddenUnits, 4 * nHiddenUnits + 3
        }); // Should be shape: [layerSize,4*layerSize+3]
    INDArray inputWeights = paramTable.get(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY);
    assertArrayEquals(
        inputWeights.shape(),
        new int[] {nIn, 4 * nHiddenUnits}); // Should be shape: [nIn,4*layerSize]
    INDArray biases = paramTable.get(GravesLSTMParamInitializer.BIAS_KEY);
    assertArrayEquals(
        biases.shape(), new int[] {1, 4 * nHiddenUnits}); // Should be shape: [1,4*layerSize]

    // Want forget gate biases to be initialized to > 0. See parameter initializer for details
    INDArray forgetGateBiases =
        biases.get(new INDArrayIndex[] {NDArrayIndex.interval(nHiddenUnits, 2 * nHiddenUnits)});
    assertTrue(forgetGateBiases.gt(0).sum(Integer.MAX_VALUE).getDouble(0) == nHiddenUnits);

    int nParams = recurrentWeights.length() + inputWeights.length() + biases.length();
    assertTrue(nParams == layer.numParams());
  }
 @Test
 public void testFeedForwardActivationsAndDerivatives() {
   MultiLayerNetwork network = new MultiLayerNetwork(getConf());
   network.init();
   DataSet data = new IrisDataSetIterator(1, 150).next();
   network.fit(data);
   Pair result = network.feedForwardActivationsAndDerivatives();
   List<INDArray> first = (List) result.getFirst();
   List<INDArray> second = (List) result.getSecond();
   assertEquals(first.size(), second.size());
 }
  @Test
  public void testMultiLayerNinNout() {
    // [Nin,Nout]: [4,13], [13,3]
    int[] hiddenLayerSizes1 = {13};
    int[] expNin1 = {4, 13};
    int[] expNout1 = {13, 3};
    MultiLayerConfiguration conf1 = getNinNoutIrisConfig(hiddenLayerSizes1);
    MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
    net1.init();
    checkNinNoutForEachLayer(expNin1, expNout1, conf1, net1);

    int[] hiddenLayerSizes2 = {5, 7};
    int[] expNin2 = {4, 5, 7};
    int[] expNout2 = {5, 7, 3};
    MultiLayerConfiguration conf2 = getNinNoutIrisConfig(hiddenLayerSizes2);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();
    checkNinNoutForEachLayer(expNin2, expNout2, conf2, net2);

    int[] hiddenLayerSizes3 = {5, 7, 9};
    int[] expNin3 = {4, 5, 7, 9};
    int[] expNout3 = {5, 7, 9, 3};
    MultiLayerConfiguration conf3 = getNinNoutIrisConfig(hiddenLayerSizes3);
    MultiLayerNetwork net3 = new MultiLayerNetwork(conf3);
    net3.init();
    checkNinNoutForEachLayer(expNin3, expNout3, conf3, net3);

    int[] hiddenLayerSizes4 = {5, 7, 9, 11, 13, 15, 17};
    int[] expNin4 = {4, 5, 7, 9, 11, 13, 15, 17};
    int[] expNout4 = {5, 7, 9, 11, 13, 15, 17, 3};
    MultiLayerConfiguration conf4 = getNinNoutIrisConfig(hiddenLayerSizes4);
    MultiLayerNetwork net4 = new MultiLayerNetwork(conf4);
    net4.init();
    checkNinNoutForEachLayer(expNin4, expNout4, conf4, net4);
  }
  @Test
  public void testGRUInit() {
    int nIn = 8;
    int nOut = 25;
    int nHiddenUnits = 17;
    MultiLayerConfiguration conf =
        new NeuralNetConfiguration.Builder()
            .list(2)
            .layer(
                0,
                new org.deeplearning4j.nn.conf.layers.GRU.Builder()
                    .nIn(nIn)
                    .nOut(nHiddenUnits)
                    .weightInit(WeightInit.DISTRIBUTION)
                    .build())
            .layer(
                1,
                new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS)
                    .nIn(nHiddenUnits)
                    .nOut(nOut)
                    .weightInit(WeightInit.DISTRIBUTION)
                    .build())
            .build();
    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    // Ensure that we have the correct number weights and biases, that these have correct shape etc.
    Layer layer = network.getLayer(0);
    assertTrue(layer instanceof GRU);

    Map<String, INDArray> paramTable = layer.paramTable();
    assertTrue(paramTable.size() == 3); // 2 sets of weights, 1 set of biases

    INDArray recurrentWeights = paramTable.get(GRUParamInitializer.RECURRENT_WEIGHT_KEY);
    assertArrayEquals(
        recurrentWeights.shape(),
        new int[] {nHiddenUnits, 3 * nHiddenUnits}); // Should be shape: [layerSize,3*layerSize]
    INDArray inputWeights = paramTable.get(GRUParamInitializer.INPUT_WEIGHT_KEY);
    assertArrayEquals(
        inputWeights.shape(),
        new int[] {nIn, 3 * nHiddenUnits}); // Should be shape: [nIn,3*layerSize]
    INDArray biases = paramTable.get(GRUParamInitializer.BIAS_KEY);
    assertArrayEquals(
        biases.shape(), new int[] {1, 3 * nHiddenUnits}); // Should be shape: [1,3*layerSize]

    int nParams = recurrentWeights.length() + inputWeights.length() + biases.length();
    assertTrue(nParams == layer.numParams());
  }
  @Test
  public void testBackProp() {
    Nd4j.getRandom().setSeed(123);
    MultiLayerConfiguration conf =
        new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT)
            .iterations(10)
            .weightInit(WeightInit.XAVIER)
            .dist(new UniformDistribution(0, 1))
            .activationFunction("tanh")
            .nIn(4)
            .nOut(3)
            .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer())
            .list(3)
            .backward(true)
            .pretrain(false)
            .hiddenLayerSizes(new int[] {3, 2})
            .override(
                2,
                new ConfOverride() {
                  @Override
                  public void overrideLayer(int i, NeuralNetConfiguration.Builder builder) {
                    builder.activationFunction("softmax");
                    builder.layer(new org.deeplearning4j.nn.conf.layers.OutputLayer());
                    builder.lossFunction(LossFunctions.LossFunction.MCXENT);
                  }
                })
            .build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    network.setListeners(Lists.<IterationListener>newArrayList(new ScoreIterationListener(1)));

    DataSetIterator iter = new IrisDataSetIterator(150, 150);

    DataSet next = iter.next();
    next.normalizeZeroMeanZeroUnitVariance();
    SplitTestAndTrain trainTest = next.splitTestAndTrain(110);
    network.setInput(trainTest.getTrain().getFeatureMatrix());
    network.setLabels(trainTest.getTrain().getLabels());
    network.init();
    network.fit(trainTest.getTrain());

    DataSet test = trainTest.getTest();
    Evaluation eval = new Evaluation();
    INDArray output = network.output(test.getFeatureMatrix());
    eval.eval(test.getLabels(), output);
    log.info("Score " + eval.stats());
  }
  @Test
  public void testSetParams() {
    Nd4j.MAX_ELEMENTS_PER_SLICE = Integer.MAX_VALUE;
    Nd4j.MAX_SLICES_TO_PRINT = Integer.MAX_VALUE;

    MultiLayerConfiguration conf =
        new NeuralNetConfiguration.Builder()
            .nIn(4)
            .nOut(3)
            .layer(new org.deeplearning4j.nn.conf.layers.RBM())
            .activationFunction("tanh")
            .list(2)
            .hiddenLayerSizes(3)
            .override(1, new ClassifierOverride(1))
            .build();

    MultiLayerNetwork network3 = new MultiLayerNetwork(conf);
    network3.init();

    INDArray params = network3.params();
    network3.setParameters(params);
    INDArray params4 = network3.params();
    assertEquals(params, params4);
  }
  private static void checkNinNoutForEachLayer(
      int[] expNin, int[] expNout, MultiLayerConfiguration conf, MultiLayerNetwork network) {

    // Check configuration
    for (int i = 0; i < expNin.length; i++) {
      NeuralNetConfiguration layerConf = conf.getConf(i);
      assertTrue(layerConf.getNIn() == expNin[i]);
      assertTrue(layerConf.getNOut() == expNout[i]);
    }

    // Check Layer
    for (int i = 0; i < expNin.length; i++) {
      Layer layer = network.getLayers()[i];
      assertTrue(layer.conf().getNIn() == expNin[i]);
      assertTrue(layer.conf().getNOut() == expNout[i]);
      int[] weightShape = layer.getParam(DefaultParamInitializer.WEIGHT_KEY).shape();
      assertTrue(weightShape[0] == expNin[i]);
      assertTrue(weightShape[1] == expNout[i]);
    }
  }
  @Test
  public void testDbn() throws Exception {
    Nd4j.MAX_SLICES_TO_PRINT = -1;
    Nd4j.MAX_ELEMENTS_PER_SLICE = -1;
    MultiLayerConfiguration conf =
        new NeuralNetConfiguration.Builder()
            .iterations(100)
            .layer(new org.deeplearning4j.nn.conf.layers.RBM())
            .weightInit(WeightInit.DISTRIBUTION)
            .dist(new UniformDistribution(0, 1))
            .activationFunction("tanh")
            .momentum(0.9)
            .optimizationAlgo(OptimizationAlgorithm.LBFGS)
            .constrainGradientToUnitNorm(true)
            .k(1)
            .regularization(true)
            .l2(2e-4)
            .visibleUnit(org.deeplearning4j.nn.conf.layers.RBM.VisibleUnit.GAUSSIAN)
            .hiddenUnit(org.deeplearning4j.nn.conf.layers.RBM.HiddenUnit.RECTIFIED)
            .lossFunction(LossFunctions.LossFunction.RMSE_XENT)
            .nIn(4)
            .nOut(3)
            .list(2)
            .hiddenLayerSizes(3)
            .override(1, new ClassifierOverride(1))
            .build();

    NeuralNetConfiguration conf2 =
        new NeuralNetConfiguration.Builder()
            .layer(new org.deeplearning4j.nn.conf.layers.RBM())
            .nIn(784)
            .nOut(600)
            .applySparsity(true)
            .sparsity(0.1)
            .build();

    Layer l =
        LayerFactories.getFactory(conf2)
            .create(conf2, Arrays.<IterationListener>asList(new ScoreIterationListener(2)), 0);

    MultiLayerNetwork d = new MultiLayerNetwork(conf);

    DataSetIterator iter = new IrisDataSetIterator(150, 150);

    DataSet next = iter.next();

    Nd4j.writeTxt(next.getFeatureMatrix(), "iris.txt", "\t");

    next.normalizeZeroMeanZeroUnitVariance();

    SplitTestAndTrain testAndTrain = next.splitTestAndTrain(110);
    DataSet train = testAndTrain.getTrain();

    d.fit(train);

    DataSet test = testAndTrain.getTest();

    Evaluation eval = new Evaluation();
    INDArray output = d.output(test.getFeatureMatrix());
    eval.eval(test.getLabels(), output);
    log.info("Score " + eval.stats());
  }