/**
* Negative log likelihood of the current input given the corruption level
*
* @return the negative log likelihood of the auto encoder given the corruption level
*/
public double negativeLoglikelihood(DoubleMatrix input) {
DoubleMatrix z = this.reconstruct(input);
if (this.useRegularization) {
double reg = (2 / l2) * MatrixFunctions.pow(this.W, 2).sum();
return -input.mul(log(z)).add(oneMinus(input).mul(log(oneMinus(z)))).columnSums().mean()
+ reg;
}
return -input.mul(log(z)).add(oneMinus(input).mul(log(oneMinus(z)))).columnSums().mean();
}
private void modifyWeights(
DoubleMatrix trainingExample,
DoubleMatrix result,
DoubleMatrix output,
double learningFactor,
double momentum,
List<DoubleMatrix> previousModifications,
EvaluationContext evalCtx) {
List<DoubleMatrix> errors = countErrors(result, output);
List<Layer> layers = getLayers();
evalCtx.resetContext();
Iterator<ActivationFunction> activationFunctionIter = evalCtx.getActivationFunction();
DoubleMatrix temporalResult = trainingExample;
for (int i = 0; i < errors.size(); i++) {
DoubleMatrix error = errors.get(i);
int layerIndex = i + 1;
Layer layer = layers.get(layerIndex);
DoubleMatrix previousModification = previousModifications.get(i);
ActivationFunction activationFunc = activationFunctionIter.next();
ActivationFunctionDerivative derivative =
DerivativeFactory.getInstance().getDerivative(activationFunc);
if (layer.includeBias()) {
temporalResult = addBiasInput(temporalResult);
}
DoubleMatrix oldVal = temporalResult.dup();
temporalResult = layer.getWeights().mmul(temporalResult);
temporalResult = activationFunc.eval(temporalResult);
// dla kazdego neuronu w warstwie
for (int j = 0; j < layer.getWeights().rows; j++) {
double derVal = derivative.evaluate(temporalResult.get(j));
DoubleMatrix oldDelta = previousModification.getRow(j);
DoubleMatrix delta = oldVal.mul(derVal).mul(learningFactor).mul(error.get(j));
delta = delta.transpose();
delta = delta.add(oldDelta.mul(momentum));
previousModification.putRow(j, delta);
DoubleMatrix oldWeights = layer.getWeights().getRow(j);
DoubleMatrix newWeights = oldWeights.add(delta);
layer.getWeights().putRow(j, newWeights);
}
}
}
/**
* Applies sparsity to the passed in hbias gradient
*
* @param hBiasGradient the hbias gradient to apply to
* @param learningRate the learning rate used
*/
protected void applySparsity(DoubleMatrix hBiasGradient, double learningRate) {
if (useAdaGrad) {
DoubleMatrix change =
this.hBiasAdaGrad
.getLearningRates(hBias)
.neg()
.mul(sparsity)
.mul(hBiasGradient.mul(sparsity));
hBiasGradient.addi(change);
} else {
DoubleMatrix change = hBiasGradient.mul(sparsity).mul(-learningRate * sparsity);
hBiasGradient.addi(change);
}
}
public DoubleMatrix free_energy(DoubleMatrix v_sample) {
DoubleMatrix wv_hb =
weights.mmul(v_sample.transpose()).addi(this.hbias.repmat(v_sample.rows, 1).transpose());
DoubleMatrix vb = v_sample.mmul(this.vbias.transpose());
DoubleMatrix hi = MatrixMath.sum(MatrixMath.log(MatrixMath.exp(wv_hb).addi(1.0)), 1);
return hi.mul(-1.0).subi(vb);
}
/**
* Negative log likelihood of the current input given the corruption level
*
* @return the negative log likelihood of the auto encoder given the corruption level
*/
@Override
public double negativeLogLikelihood() {
DoubleMatrix z = this.reconstruct(input);
if (this.useRegularization) {
double reg = (2 / l2) * MatrixFunctions.pow(this.W, 2).sum();
double ret =
-input.mul(log(z)).add(oneMinus(input).mul(log(oneMinus(z)))).columnSums().mean() + reg;
if (this.normalizeByInputRows) ret /= input.rows;
return ret;
}
double likelihood =
-input.mul(log(z)).add(oneMinus(input).mul(log(oneMinus(z)))).columnSums().mean();
if (this.normalizeByInputRows) likelihood /= input.rows;
return likelihood;
}
public void weight_contribution(
DoubleMatrix h0, DoubleMatrix v0, DoubleMatrix h1, DoubleMatrix v1) {
this.w1
.addi(h0.transpose().mmul(v0).subi(h1.transpose().mmul(v1)))
.muli(learningRate / v_data.rows)
.subi(weights.mul(weightCost));
this.vb1.addi(v0.sub(v1).muli(learningRate / v_data.rows).columnMeans());
this.hb1.addi(h0.sub(h1).muli(learningRate / v_data.rows).columnMeans());
}
/**
* Reconstruction entropy. This compares the similarity of two probability distributions, in this
* case that would be the input and the reconstructed input with gaussian noise. This will account
* for either regularization or none depending on the configuration.
*
* @return reconstruction error
*/
public double getReConstructionCrossEntropy() {
DoubleMatrix preSigH = input.mmul(W).addRowVector(hBias);
DoubleMatrix sigH = sigmoid(preSigH);
DoubleMatrix preSigV = sigH.mmul(W.transpose()).addRowVector(vBias);
DoubleMatrix sigV = sigmoid(preSigV);
DoubleMatrix inner = input.mul(log(sigV)).add(oneMinus(input).mul(log(oneMinus(sigV))));
double ret = inner.rowSums().mean();
if (normalizeByInputRows) ret /= input.rows;
return ret;
}
@Test
public void testWithMnist() throws Exception {
MnistDataFetcher fetcher = new MnistDataFetcher(true);
fetcher.fetch(200);
DataSet data = fetcher.next();
data.filterAndStrip(new int[] {0, 1});
log.info("Training on " + data.numExamples());
DBN dbn =
new DBN.Builder()
.hiddenLayerSizes(new int[] {1000, 500, 250, 10})
.numberOfInputs(784)
.numberOfOutPuts(2)
.build();
dbn.pretrain(data.getFirst(), new Object[] {1, 1e-1, 10000});
DeepAutoEncoder encoder = new DeepAutoEncoder(dbn);
encoder.finetune(data.getFirst(), 1e-3, 1000);
DoubleMatrix reconstruct = encoder.reconstruct(data.getFirst());
for (int j = 0; j < data.numExamples(); j++) {
DoubleMatrix draw1 = data.get(j).getFirst().mul(255);
DoubleMatrix reconstructed2 = reconstruct.getRow(j);
DoubleMatrix draw2 = reconstructed2.mul(255);
DrawMnistGreyScale d = new DrawMnistGreyScale(draw1);
d.title = "REAL";
d.draw();
DrawMnistGreyScale d2 = new DrawMnistGreyScale(draw2);
d2.title = "TEST";
d2.draw();
Thread.sleep(10000);
d.frame.dispose();
d2.frame.dispose();
}
}
private void costantiniUnwrap() throws LPException {
final int ny = wrappedPhase.rows - 1; // start from Zero!
final int nx = wrappedPhase.columns - 1; // start from Zero!
if (wrappedPhase.isVector()) throw new IllegalArgumentException("Input must be 2D array");
if (wrappedPhase.rows < 2 || wrappedPhase.columns < 2)
throw new IllegalArgumentException("Size of input must be larger than 2");
// Default weight
DoubleMatrix w1 = DoubleMatrix.ones(ny + 1, 1);
w1.put(0, 0.5);
w1.put(w1.length - 1, 0.5);
DoubleMatrix w2 = DoubleMatrix.ones(1, nx + 1);
w2.put(0, 0.5);
w2.put(w2.length - 1, 0.5);
DoubleMatrix weight = w1.mmul(w2);
DoubleMatrix i, j, I_J, IP1_J, I_JP1;
DoubleMatrix Psi1, Psi2;
DoubleMatrix[] ROWS;
// Compute partial derivative Psi1, eqt (1,3)
i = intRangeDoubleMatrix(0, ny - 1);
j = intRangeDoubleMatrix(0, nx);
ROWS = grid2D(i, j);
I_J = JblasUtils.sub2ind(wrappedPhase.rows, ROWS[0], ROWS[1]);
IP1_J = JblasUtils.sub2ind(wrappedPhase.rows, ROWS[0].add(1), ROWS[1]);
Psi1 =
JblasUtils.getMatrixFromIdx(wrappedPhase, IP1_J)
.sub(JblasUtils.getMatrixFromIdx(wrappedPhase, I_J));
Psi1 = UnwrapUtils.wrapDoubleMatrix(Psi1);
// Compute partial derivative Psi2, eqt (2,4)
i = intRangeDoubleMatrix(0, ny);
j = intRangeDoubleMatrix(0, nx - 1);
ROWS = grid2D(i, j);
I_J = JblasUtils.sub2ind(wrappedPhase.rows, ROWS[0], ROWS[1]);
I_JP1 = JblasUtils.sub2ind(wrappedPhase.rows, ROWS[0], ROWS[1].add(1));
Psi2 =
JblasUtils.getMatrixFromIdx(wrappedPhase, I_JP1)
.sub(JblasUtils.getMatrixFromIdx(wrappedPhase, I_J));
Psi2 = UnwrapUtils.wrapDoubleMatrix(Psi2);
// Compute beq
DoubleMatrix beq = DoubleMatrix.zeros(ny, nx);
i = intRangeDoubleMatrix(0, ny - 1);
j = intRangeDoubleMatrix(0, nx - 1);
ROWS = grid2D(i, j);
I_J = JblasUtils.sub2ind(Psi1.rows, ROWS[0], ROWS[1]);
I_JP1 = JblasUtils.sub2ind(Psi1.rows, ROWS[0], ROWS[1].add(1));
beq.addi(JblasUtils.getMatrixFromIdx(Psi1, I_JP1).sub(JblasUtils.getMatrixFromIdx(Psi1, I_J)));
I_J = JblasUtils.sub2ind(Psi2.rows, ROWS[0], ROWS[1]);
I_JP1 = JblasUtils.sub2ind(Psi2.rows, ROWS[0].add(1), ROWS[1]);
beq.subi(JblasUtils.getMatrixFromIdx(Psi2, I_JP1).sub(JblasUtils.getMatrixFromIdx(Psi2, I_J)));
beq.muli(-1 / (2 * Constants._PI));
for (int k = 0; k < beq.length; k++) {
beq.put(k, Math.round(beq.get(k)));
}
beq.reshape(beq.length, 1);
logger.debug("Constraint matrix");
i = intRangeDoubleMatrix(0, ny - 1);
j = intRangeDoubleMatrix(0, nx - 1);
ROWS = grid2D(i, j);
DoubleMatrix ROW_I_J = JblasUtils.sub2ind(i.length, ROWS[0], ROWS[1]);
int nS0 = nx * ny;
// Use by S1p, S1m
DoubleMatrix[] COLS;
COLS = grid2D(i, j);
DoubleMatrix COL_IJ_1 = JblasUtils.sub2ind(i.length, COLS[0], COLS[1]);
COLS = grid2D(i, j.add(1));
DoubleMatrix COL_I_JP1 = JblasUtils.sub2ind(i.length, COLS[0], COLS[1]);
int nS1 = (nx + 1) * (ny);
// SOAPBinding.Use by S2p, S2m
COLS = grid2D(i, j);
DoubleMatrix COL_IJ_2 = JblasUtils.sub2ind(i.length + 1, COLS[0], COLS[1]);
COLS = grid2D(i.add(1), j);
DoubleMatrix COL_IP1_J = JblasUtils.sub2ind(i.length + 1, COLS[0], COLS[1]);
int nS2 = nx * (ny + 1);
// Equality constraint matrix (Aeq)
/*
S1p = + sparse(ROW_I_J, COL_I_JP1,1,nS0,nS1) ...
- sparse(ROW_I_J, COL_IJ_1,1,nS0,nS1);
S1m = -S1p;
S2p = - sparse(ROW_I_J, COL_IP1_J,1,nS0,nS2) ...
+ sparse(ROW_I_J, COL_IJ_2,1,nS0,nS2);
S2m = -S2p;
*/
// ToDo: Aeq matrix should be sparse from it's initialization, look into JblasMatrix factory for
// howto
// ...otherwise even a data set of eg 40x40 pixels will exhaust heap:
// ... dimension of Aeq (equality constraints) matrix for 30x30 input is 1521x6240 matrix
// ... dimension of Aeq ( ) matrix for 50x50 input is 2401x9800
// ... dimension of Aeq ( ) matrix for 512x512 input is 261121x1046528
DoubleMatrix S1p =
JblasUtils.setUpMatrixFromIdx(nS0, nS1, ROW_I_J, COL_I_JP1)
.sub(JblasUtils.setUpMatrixFromIdx(nS0, nS1, ROW_I_J, COL_IJ_1));
DoubleMatrix S1m = S1p.neg();
DoubleMatrix S2p =
JblasUtils.setUpMatrixFromIdx(nS0, nS2, ROW_I_J, COL_IP1_J)
.neg()
.add(JblasUtils.setUpMatrixFromIdx(nS0, nS2, ROW_I_J, COL_IJ_2));
DoubleMatrix S2m = S2p.neg();
DoubleMatrix Aeq =
concatHorizontally(concatHorizontally(S1p, S1m), concatHorizontally(S2p, S2m));
final int nObs = Aeq.columns;
final int nUnkn = Aeq.rows;
DoubleMatrix c1 = JblasUtils.getMatrixFromRange(0, ny, 0, weight.columns, weight);
DoubleMatrix c2 = JblasUtils.getMatrixFromRange(0, weight.rows, 0, nx, weight);
c1.reshape(c1.length, 1);
c2.reshape(c2.length, 1);
DoubleMatrix cost =
DoubleMatrix.concatVertically(
DoubleMatrix.concatVertically(c1, c1), DoubleMatrix.concatVertically(c2, c2));
logger.debug("Minimum network flow resolution");
StopWatch clockLP = new StopWatch();
LinearProgram lp = new LinearProgram(cost.data);
lp.setMinProblem(true);
boolean[] integerBool = new boolean[nObs];
double[] lowerBound = new double[nObs];
double[] upperBound = new double[nObs];
for (int k = 0; k < nUnkn; k++) {
lp.addConstraint(new LinearEqualsConstraint(Aeq.getRow(k).toArray(), beq.get(k), "cost"));
}
for (int k = 0; k < nObs; k++) {
integerBool[k] = true;
lowerBound[k] = 0;
upperBound[k] = 99999;
}
// setup bounds and integer nature
lp.setIsinteger(integerBool);
lp.setUpperbound(upperBound);
lp.setLowerbound(lowerBound);
LinearProgramSolver solver = SolverFactory.newDefault();
// double[] solution;
// solution = solver.solve(lp);
DoubleMatrix solution = new DoubleMatrix(solver.solve(lp));
clockLP.stop();
logger.debug("Total GLPK time: {} [sec]", (double) (clockLP.getElapsedTime()) / 1000);
// Displatch the LP solution
int offset;
int[] idx1p = JblasUtils.intRangeIntArray(0, nS1 - 1);
DoubleMatrix x1p = solution.get(idx1p);
x1p.reshape(ny, nx + 1);
offset = idx1p[nS1 - 1] + 1;
int[] idx1m = JblasUtils.intRangeIntArray(offset, offset + nS1 - 1);
DoubleMatrix x1m = solution.get(idx1m);
x1m.reshape(ny, nx + 1);
offset = idx1m[idx1m.length - 1] + 1;
int[] idx2p = JblasUtils.intRangeIntArray(offset, offset + nS2 - 1);
DoubleMatrix x2p = solution.get(idx2p);
x2p.reshape(ny + 1, nx);
offset = idx2p[idx2p.length - 1] + 1;
int[] idx2m = JblasUtils.intRangeIntArray(offset, offset + nS2 - 1);
DoubleMatrix x2m = solution.get(idx2m);
x2m.reshape(ny + 1, nx);
// Compute the derivative jumps, eqt (20,21)
DoubleMatrix k1 = x1p.sub(x1m);
DoubleMatrix k2 = x2p.sub(x2m);
// (?) Round to integer solution
if (roundK == true) {
for (int idx = 0; idx < k1.length; idx++) {
k1.put(idx, FastMath.floor(k1.get(idx)));
}
for (int idx = 0; idx < k2.length; idx++) {
k2.put(idx, FastMath.floor(k2.get(idx)));
}
}
// Sum the jumps with the wrapped partial derivatives, eqt (10,11)
k1.reshape(ny, nx + 1);
k2.reshape(ny + 1, nx);
k1.addi(Psi1.div(Constants._TWO_PI));
k2.addi(Psi2.div(Constants._TWO_PI));
// Integrate the partial derivatives, eqt (6)
// cumsum() method in JblasTester -> see cumsum_demo() in JblasTester.cumsum_demo()
DoubleMatrix k2_temp = DoubleMatrix.concatHorizontally(DoubleMatrix.zeros(1), k2.getRow(0));
k2_temp = JblasUtils.cumsum(k2_temp, 1);
DoubleMatrix k = DoubleMatrix.concatVertically(k2_temp, k1);
k = JblasUtils.cumsum(k, 1);
// Unwrap - final solution
unwrappedPhase = k.mul(Constants._TWO_PI);
}
/**
* Update the gradient according to the configuration such as adagrad, momentum, and sparsity
*
* @param gradient the gradient to modify
* @param iteration the current iteration
* @param learningRate the learning rate for the current iteration
*/
protected void updateGradientAccordingToParams(
NeuralNetworkGradient gradient, int iteration, double learningRate) {
DoubleMatrix wGradient = gradient.getwGradient();
DoubleMatrix hBiasGradient = gradient.gethBiasGradient();
DoubleMatrix vBiasGradient = gradient.getvBiasGradient();
// reset adagrad history
if (iteration != 0 && resetAdaGradIterations > 0 && iteration % resetAdaGradIterations == 0) {
wAdaGrad.historicalGradient = null;
hBiasAdaGrad.historicalGradient = null;
vBiasAdaGrad.historicalGradient = null;
if (this.W != null && this.wAdaGrad == null)
this.wAdaGrad = new AdaGrad(this.W.rows, this.W.columns);
if (this.vBias != null && this.vBiasAdaGrad == null)
this.vBiasAdaGrad = new AdaGrad(this.vBias.rows, this.vBias.columns);
if (this.hBias != null && this.hBiasAdaGrad == null)
this.hBiasAdaGrad = new AdaGrad(this.hBias.rows, this.hBias.columns);
log.info("Resetting adagrad");
}
DoubleMatrix wLearningRates = wAdaGrad.getLearningRates(wGradient);
// change up momentum after so many iterations if specified
double momentum = this.momentum;
if (momentumAfter != null && !momentumAfter.isEmpty()) {
int key = momentumAfter.keySet().iterator().next();
if (iteration >= key) {
momentum = momentumAfter.get(key);
}
}
if (useAdaGrad) wGradient.muli(wLearningRates);
else wGradient.muli(learningRate);
if (useAdaGrad) hBiasGradient = hBiasGradient.mul(hBiasAdaGrad.getLearningRates(hBiasGradient));
else hBiasGradient = hBiasGradient.mul(learningRate);
if (useAdaGrad) vBiasGradient = vBiasGradient.mul(vBiasAdaGrad.getLearningRates(vBiasGradient));
else vBiasGradient = vBiasGradient.mul(learningRate);
// only do this with binary hidden layers
if (applySparsity && this.hBiasGradient != null) applySparsity(hBiasGradient, learningRate);
if (momentum != 0 && this.wGradient != null)
wGradient.addi(this.wGradient.mul(momentum).add(wGradient.mul(1 - momentum)));
if (momentum != 0 && this.vBiasGradient != null)
vBiasGradient.addi(this.vBiasGradient.mul(momentum).add(vBiasGradient.mul(1 - momentum)));
if (momentum != 0 && this.hBiasGradient != null)
hBiasGradient.addi(this.hBiasGradient.mul(momentum).add(hBiasGradient.mul(1 - momentum)));
if (normalizeByInputRows) {
wGradient.divi(lastMiniBatchSize);
vBiasGradient.divi(lastMiniBatchSize);
hBiasGradient.divi(lastMiniBatchSize);
}
// simulate post gradient application and apply the difference to the gradient to decrease the
// change the gradient has
if (useRegularization && l2 > 0) {
if (useAdaGrad) wGradient.subi(W.mul(l2).mul(wLearningRates));
else wGradient.subi(W.mul(l2 * learningRate));
}
if (constrainGradientToUnitNorm) {
wGradient.divi(wGradient.norm2());
vBiasGradient.divi(vBiasGradient.norm2());
hBiasGradient.divi(hBiasGradient.norm2());
}
this.wGradient = wGradient;
this.vBiasGradient = vBiasGradient;
this.hBiasGradient = hBiasGradient;
}
