/**
* Get a vector orthogonal to the instance.
*
* <p>There are an infinite number of normalized vectors orthogonal to the instance. This method
* picks up one of them almost arbitrarily. It is useful when one needs to compute a reference
* frame with one of the axes in a predefined direction. The following example shows how to build
* a frame having the k axis aligned with the known vector u :
*
* <pre><code>
* Vector3D k = u.normalize();
* Vector3D i = k.orthogonal();
* Vector3D j = Vector3D.crossProduct(k, i);
* </code></pre>
*
* @return a new normalized vector orthogonal to the instance
* @exception ArithmeticException if the norm of the instance is null
*/
public Vector3D orthogonal() {
double threshold = 0.6 * getNorm();
if (threshold == 0) {
throw MathRuntimeException.createArithmeticException(LocalizedFormats.ZERO_NORM);
}
if ((x >= -threshold) && (x <= threshold)) {
double inverse = 1 / FastMath.sqrt(y * y + z * z);
return new Vector3D(0, inverse * z, -inverse * y);
} else if ((y >= -threshold) && (y <= threshold)) {
double inverse = 1 / FastMath.sqrt(x * x + z * z);
return new Vector3D(-inverse * z, 0, inverse * x);
}
double inverse = 1 / FastMath.sqrt(x * x + y * y);
return new Vector3D(inverse * y, -inverse * x, 0);
}
/**
* Returns the next pseudorandom, Gaussian ("normally") distributed {@code double} value with mean
* {@code 0.0} and standard deviation {@code 1.0} from this random number generator's sequence.
*
* <p>The default implementation uses the <em>Polar Method</em> due to G.E.P. Box, M.E. Muller and
* G. Marsaglia, as described in D. Knuth, <u>The Art of Computer Programming</u>, 3.4.1C.
*
* <p>The algorithm generates a pair of independent random values. One of these is cached for
* reuse, so the full algorithm is not executed on each activation. Implementations that do not
* override this method should make sure to call {@link #clear} to clear the cached value in the
* implementation of {@link #setSeed(long)}.
*
* @return the next pseudorandom, Gaussian ("normally") distributed {@code double} value with mean
* {@code 0.0} and standard deviation {@code 1.0} from this random number generator's sequence
*/
public double nextGaussian() {
if (!Double.isNaN(cachedNormalDeviate)) {
double dev = cachedNormalDeviate;
cachedNormalDeviate = Double.NaN;
return dev;
}
double v1 = 0;
double v2 = 0;
double s = 1;
while (s >= 1) {
v1 = 2 * nextDouble() - 1;
v2 = 2 * nextDouble() - 1;
s = v1 * v1 + v2 * v2;
}
if (s != 0) {
s = FastMath.sqrt(-2 * FastMath.log(s) / s);
}
cachedNormalDeviate = v2 * s;
return v1 * s;
}
/**
* Compute the distance between two vectors according to the L<sub>2</sub> norm.
*
* <p>Calling this method is equivalent to calling: <code>v1.subtract(v2).getNorm()</code> except
* that no intermediate vector is built
*
* @param v1 first vector
* @param v2 second vector
* @return the distance between v1 and v2 according to the L<sub>2</sub> norm
*/
public static double distance(Vector3D v1, Vector3D v2) {
final double dx = v2.x - v1.x;
final double dy = v2.y - v1.y;
final double dz = v2.z - v1.z;
return FastMath.sqrt(dx * dx + dy * dy + dz * dz);
}
/**
* Get the L<sub>2</sub> norm for the vector.
*
* @return euclidian norm for the vector
*/
public double getNorm() {
return FastMath.sqrt(x * x + y * y + z * z);
}
public class DenseFeatureMatrix {
int inputSize;
int outputSize;
INDArray features;
INDArray featuresT;
GradientStore gradientStore = new GradientStore();
double l2 = GlobalParameters.l2regularizerLambdaDefault;
double learningRate = GlobalParameters.learningRateDefault;
// adagrad vars
boolean useAdagrad = GlobalParameters.useAdagradDefault;
INDArray adagradQuotient;
double adagradEps = 0.001;
double adagradMax = 10;
// gaussian noise
double noiseVar = GlobalParameters.noiseDevDefault;
double noiseVarSqrt = FastMath.sqrt(noiseVar);;
HashMap<Integer, INDArray> currentNoise = new HashMap<Integer, INDArray>();
// momentum vars
boolean useMomentum = GlobalParameters.useMomentumDefault;
INDArray momentumPrevUpdate;
double momentum = GlobalParameters.momentumDefault;
// adadelta vars
boolean useAdadelta = GlobalParameters.useAdadeltaDefault;
INDArray adadeltaRMSGradient;
INDArray adadeltaRMSUpdate;
double adadeltaMomentum = GlobalParameters.adadeltaMomentumDefault;
double adadeltaEps = GlobalParameters.adadeltaEpsDefault;
// commit
int commitMethod = GlobalParameters.commitMethodDefault;
public DenseFeatureMatrix(
int inputSize, int outputSize, boolean useAdagrad, boolean useMomentum, boolean useAdadelta) {
if (inputSize == 1) {
throw new RuntimeException("input size = 1: use vector instead");
}
this.inputSize = inputSize;
this.outputSize = outputSize;
this.useAdadelta = useAdadelta;
this.useAdagrad = useAdagrad;
this.useMomentum = useMomentum;
if (useAdagrad) {
adagradQuotient = Nd4j.zeros(inputSize, outputSize);
adagradQuotient.addi(adagradEps);
}
if (useMomentum) {
momentumPrevUpdate = Nd4j.zeros(inputSize, outputSize);
}
if (useAdadelta) {
adadeltaRMSGradient = Nd4j.zeros(inputSize, outputSize);
adadeltaRMSUpdate = Nd4j.zeros(inputSize, outputSize);
}
}
public DenseFeatureMatrix(int inputSize, int outputSize) {
if (inputSize == 1) {
throw new RuntimeException("input size = 1: use vector instead");
}
this.inputSize = inputSize;
this.outputSize = outputSize;
if (useAdagrad) {
adagradQuotient = Nd4j.zeros(inputSize, outputSize);
adagradQuotient.addi(adagradEps);
}
if (useMomentum) {
momentumPrevUpdate = Nd4j.zeros(inputSize, outputSize);
}
if (useAdadelta) {
adadeltaRMSGradient = Nd4j.zeros(inputSize, outputSize);
adadeltaRMSUpdate = Nd4j.zeros(inputSize, outputSize);
}
}
public void initialize(double[][] vals) {
features = Nd4j.create(vals);
featuresT = features.transpose();
}
public void initializeUniform(double min, double max) {
double[][] featuresMatrixStub = new double[inputSize][outputSize];
RandomUtils.initializeRandomMatrix(featuresMatrixStub, min, max, 1);
features = Nd4j.create(featuresMatrixStub);
featuresT = features.transpose();
}
public void normalizedInitializationHtan(int fanin, int fanout) {
double max = Math.sqrt(6.0d / (fanout + fanin));
double[][] featuresMatrixStub = new double[inputSize][outputSize];
RandomUtils.initializeRandomMatrix(featuresMatrixStub, -max, max, 1);
features = Nd4j.create(featuresMatrixStub);
featuresT = features.transpose();
}
public void normalizedInitializationSigmoid(int fanin, int fanout) {
double max = 4 * Math.sqrt(6.0d / (fanin + fanout));
double[][] featuresMatrixStub = new double[inputSize][outputSize];
RandomUtils.initializeRandomMatrix(featuresMatrixStub, -max, max, 1);
features = Nd4j.create(featuresMatrixStub);
featuresT = features.transpose();
}
public INDArray getWeights() {
return features;
}
public INDArray getTranspose() {
return featuresT;
}
public void storeGradients(int processId, INDArray gradient) {
gradientStore.addGradient(processId, gradient);
}
public void storeInputsAndOutputs(int id, INDArray x, INDArray yGrad) {
gradientStore.addInputAndOutput(id, x, yGrad);
}
public void checkinGradients(int id) {
gradientStore.computeGradientAndAdd(id);
}
public void update() {
INDArray gradient = null;
if (commitMethod == 0) {
gradient = gradientStore.getGradientAvg();
} else {
gradient = gradientStore.getGradientSum();
}
if (gradient == null) return;
INDArray gradientL2 = gradient.sub(features.mul(l2));
if (useAdagrad) {
getAdagradGradient(gradientL2);
features.addi(gradientL2.mul(learningRate));
} else if (useMomentum) {
getMomentumGradient(gradientL2);
features.addi(gradientL2.mul(learningRate));
} else if (useAdadelta) {
getAdadeltaGradient(gradientL2);
features.addi(gradientL2);
} else {
features.addi(gradientL2.mul(learningRate));
}
capValues(GlobalParameters.maxVal);
featuresT = features.transpose();
gradientStore.init();
}
protected void getAdagradGradient(INDArray gradient) {
adagradQuotient.addi(gradient.mul(gradient));
for (int i = 0; i < inputSize; i++) {
for (int j = 0; j < outputSize; j++) {
double adagradQ = adagradQuotient.getDouble(i, j);
if (adagradMax < adagradQ) {
adagradQuotient.putScalar(new int[] {i, j}, adagradMax);
adagradQ = adagradMax;
}
gradient.putScalar(new int[] {i, j}, gradient.getDouble(i, j) / Math.sqrt(adagradQ));
}
}
}
protected void getAdadeltaGradient(INDArray gradient) {
adadeltaRMSGradient =
adadeltaRMSGradient
.mul(adadeltaMomentum)
.add(gradient.mul(gradient).mul(1 - adadeltaMomentum));
gradient.muli(
Transforms.sqrt(adadeltaRMSUpdate.add(adadeltaEps))
.div(Transforms.sqrt(adadeltaRMSGradient.add(adadeltaEps))));
adadeltaRMSUpdate.mul(adadeltaMomentum).add(gradient.mul(gradient).mul(1 - adadeltaMomentum));
}
protected void getMomentumGradient(INDArray gradient) {
INDArray momemtumUpdate = momentumPrevUpdate.mul(momentum);
gradient.addi(momemtumUpdate);
momentumPrevUpdate = gradient.dup();
}
public INDArray genGaussianNoise(int id) {
if (!currentNoise.containsKey(id)) {
INDArray zeroMean = Nd4j.zeros(inputSize, outputSize);
currentNoise.put(id, Sampling.normal(RandomUtils.getRandomGenerator(id), zeroMean, noiseVar));
} else {
RealDistribution reals =
new NormalDistribution(
RandomUtils.getRandomGenerator(id),
0,
noiseVarSqrt,
NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
INDArrayUtils.shiftLeft(
currentNoise.get(id),
inputSize,
outputSize,
RandomUtils.getRandomGenerator(id).nextInt(inputSize * outputSize),
reals.sample());
}
// currentNoise = Sampling.normal(RandomUtils.getRandomGenerator(id), zeroMean, noiseVar);
return currentNoise.get(id);
}
public void capValues(double max) {
INDArray linear = features.linearView();
for (int i = 0; i < linear.size(0); i++) {
linear.putScalar(i, Math.max(-max, Math.min(max, linear.getDouble(i))));
}
}
public void save(PrintStream out) {
out.println(inputSize);
out.println(outputSize);
out.println(useAdagrad);
out.println(adagradEps);
out.println(adagradMax);
out.println(noiseVar);
out.println(useMomentum);
out.println(momentum);
out.println(useAdadelta);
out.println(adadeltaEps);
out.println(adadeltaMomentum);
saveMatrix(features, out);
if (useAdagrad) {
saveMatrix(adagradQuotient, out);
}
if (useMomentum) {
saveMatrix(momentumPrevUpdate, out);
}
if (useAdadelta) {
saveMatrix(adadeltaRMSGradient, out);
saveMatrix(adadeltaRMSUpdate, out);
}
}
public void saveMatrix(INDArray matrix, PrintStream out) {
for (int row = 0; row < inputSize; row++) {
for (int col = 0; col < outputSize; col++) {
double val = matrix.getDouble(row, col);
if (col < outputSize - 1) {
out.print(val + " ");
} else {
out.println(val);
}
}
}
}
public static DenseFeatureMatrix load(BufferedReader in) {
try {
int inputSize = Integer.parseInt(in.readLine());
int outputSize = Integer.parseInt(in.readLine());
DenseFeatureMatrix matrix = new DenseFeatureMatrix(inputSize, outputSize);
matrix.useAdagrad = Boolean.parseBoolean(in.readLine());
matrix.adagradEps = Double.parseDouble(in.readLine());
matrix.adagradMax = Double.parseDouble(in.readLine());
matrix.noiseVar = Double.parseDouble(in.readLine());
matrix.useMomentum = Boolean.parseBoolean(in.readLine());
matrix.momentum = Double.parseDouble(in.readLine());
matrix.useAdadelta = Boolean.parseBoolean(in.readLine());
matrix.adadeltaEps = Double.parseDouble(in.readLine());
matrix.adadeltaMomentum = Double.parseDouble(in.readLine());
matrix.features = loadMatrix(in, inputSize, outputSize);
if (matrix.useAdagrad) {
matrix.adagradQuotient = loadMatrix(in, inputSize, outputSize);
}
if (matrix.useMomentum) {
matrix.momentumPrevUpdate = loadMatrix(in, inputSize, outputSize);
}
if (matrix.useAdadelta) {
matrix.adadeltaRMSGradient = loadMatrix(in, inputSize, outputSize);
matrix.adadeltaRMSUpdate = loadMatrix(in, inputSize, outputSize);
}
matrix.featuresT = matrix.features.transpose();
return matrix;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static INDArray loadMatrix(BufferedReader in, int inputSize, int outputSize)
throws IOException {
INDArray matrix = Nd4j.create(inputSize, outputSize);
for (int row = 0; row < inputSize; row++) {
String[] vals = in.readLine().split("\\s+");
for (int col = 0; col < outputSize; col++) {
matrix.putScalar(new int[] {row, col}, Double.parseDouble(vals[col]));
}
}
return matrix;
}
public void setL2(double l2) {
this.l2 = l2;
}
public static void main(String[] args) {
DenseFeatureMatrix matrix = new DenseFeatureMatrix(1, 5);
matrix.initializeUniform(-0.1, 0.1);
matrix.save(IOUtils.getPrintStream("/tmp/file"));
DenseFeatureMatrix.load(IOUtils.getReader("/tmp/file")).save(System.err);
}
public void normalize() {
features.divi(features.sum(0).getDouble(0) * inputSize);
featuresT = features.transpose();
}
}