@Test
public void testInverse2x2() {
double tol = 0.001;
Map<Key, Value> input = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
input.put(new Key("1", "", "1"), new Value("4".getBytes()));
input.put(new Key("1", "", "2"), new Value("3".getBytes()));
input.put(new Key("2", "", "1"), new Value("1".getBytes()));
input.put(new Key("2", "", "2"), new Value("1".getBytes()));
Map<Key, Value> expect = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
expect.put(new Key("1", "", "1"), new Value("1 ".getBytes()));
expect.put(new Key("1", "", "2"), new Value("-3".getBytes()));
expect.put(new Key("2", "", "1"), new Value("-1".getBytes()));
expect.put(new Key("2", "", "2"), new Value("4 ".getBytes()));
RealMatrix matrix = MemMatrixUtil.buildMatrix(input.entrySet().iterator(), 2);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(4, matrix.getEntry(0, 0), tol);
Assert.assertEquals(3, matrix.getEntry(0, 1), tol);
Assert.assertEquals(1, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
matrix = MemMatrixUtil.doInverse(matrix, -1);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(-3, matrix.getEntry(0, 1), tol);
Assert.assertEquals(-1, matrix.getEntry(1, 0), tol);
Assert.assertEquals(4, matrix.getEntry(1, 1), tol);
SortedMap<Key, Value> back =
MemMatrixUtil.matrixToMap(new TreeMap<Key, Value>(TestUtil.COMPARE_KEY_TO_COLQ), matrix);
TestUtil.assertEqualDoubleMap(expect, back);
}
/**
* Calculates {@code P(D_n < d)} using method described in [1] and doubles (see above).
*
* @param d statistic
* @return the two-sided probability of {@code P(D_n < d)}
* @throws MathArithmeticException if algorithm fails to convert {@code h} to a {@link
* org.apache.commons.math3.fraction.BigFraction} in expressing {@code d} as {@code (k - h) /
* m} for integer {@code k, m} and {@code 0 <= h < 1}.
*/
private double roundedK(double d) throws MathArithmeticException {
final int k = (int) FastMath.ceil(n * d);
final FieldMatrix<BigFraction> HBigFraction = this.createH(d);
final int m = HBigFraction.getRowDimension();
/*
* Here the rounding part comes into play: use
* RealMatrix instead of FieldMatrix<BigFraction>
*/
final RealMatrix H = new Array2DRowRealMatrix(m, m);
for (int i = 0; i < m; ++i) {
for (int j = 0; j < m; ++j) {
H.setEntry(i, j, HBigFraction.getEntry(i, j).doubleValue());
}
}
final RealMatrix Hpower = H.power(n);
double pFrac = Hpower.getEntry(k - 1, k - 1);
for (int i = 1; i <= n; ++i) {
pFrac *= (double) i / (double) n;
}
return pFrac;
}
private static Intersection getIntersection(Ray ray, SphereObject obj, Model model) {
RealMatrix transform = obj.getTransform();
final RealMatrix transformInverse = obj.getTransformInverse();
ray = ray.transform(transformInverse);
Vector3D c = VectorUtils.toVector3D(obj.getCenter());
Vector3D p0 = VectorUtils.toVector3D(ray.getP0());
Vector3D p1 = VectorUtils.toVector3D(ray.getP1());
float a = (float) p1.dotProduct(p1);
Vector3D p0c = p0.subtract(c);
float b = (float) p1.dotProduct(p0c) * 2.0f;
float cc = (float) (p0c.dotProduct(p0c)) - obj.getSize() * obj.getSize();
Double t = quadraticEquationRoot1(a, b, cc);
if (t == null || t < EPSILON) {
return new Intersection(false);
}
Intersection result = new Intersection(true);
result.setObject(obj);
final Vector3D p = p0.add(p1.scalarMultiply(t));
RealVector pv = VectorUtils.toRealVector(p);
pv.setEntry(3, 1.0);
RealVector pt = transform.preMultiply(pv);
result.setP(VectorUtils.toVector3D(pt));
RealVector nv = pv.subtract(obj.getCenter());
final RealVector nvt = transformInverse.transpose().preMultiply(nv);
result.setN(VectorUtils.toVector3D(nvt).normalize());
result.setDistance(t);
return result;
}
@Override
public Label predict(Instance instance) {
Label l = null;
if (instance.getLabel() instanceof ClassificationLabel || instance.getLabel() == null) {
// ----------------- declare variables ------------------
double lambda = 0.0;
RealVector x_instance = new ArrayRealVector(matrixX.getColumnDimension(), 0);
double result = 0.0;
// -------------------------- initialize xi -------------------------
for (int idx = 0; idx < matrixX.getColumnDimension(); idx++) {
x_instance.setEntry(idx, instance.getFeatureVector().get(idx + 1));
}
// ------------------ get lambda -----------------------
for (int j = 0; j < alpha.getDimension(); j++) {
lambda += alpha.getEntry(j) * kernelFunction(matrixX.getRowVector(j), x_instance);
}
// ----------------- make prediction -----------------
Sigmoid g = new Sigmoid(); // helper function
result = g.value(lambda);
l = new ClassificationLabel(result < 0.5 ? 0 : 1);
} else {
System.out.println("label type error!");
}
return l;
}
/**
* @param weight Weight matrix.
* @throws NonSquareMatrixException if the argument is not a square matrix.
*/
public Weight(RealMatrix weight) {
if (weight.getColumnDimension() != weight.getRowDimension()) {
throw new NonSquareMatrixException(weight.getColumnDimension(), weight.getRowDimension());
}
weightMatrix = weight.copy();
}
public static RealMatrix stochasticSubmatrix(RealMatrix data, int batch_size, Random rng) {
// assume all data has the size number_samples by number_features
int num_samples = data.getRowDimension();
int num_features = data.getColumnDimension();
int batch_num = num_samples / batch_size + 1;
// randomly generate a batch index
int batch_index = rng.nextInt(batch_num);
List<Integer> rowIndex_tmp = new ArrayList<Integer>();
for (int i = 0; i < batch_size; i++) {
if (batch_size * batch_index + i >= num_samples) {
break;
} else {
rowIndex_tmp.add(batch_size * batch_index + i);
}
}
int[] rowIndex = TypeConvert.ArrayTointv(rowIndex_tmp);
// System.out.println(rowIndex_tmp);
int[] columnIndex = new int[num_features];
for (int j = 0; j < num_features; j++) {
columnIndex[j] = j;
}
// System.out.println(batch_index);
// return null;
return data.getSubMatrix(rowIndex, columnIndex);
}
/**
* Build a rating matrix from the rating data. Each user's ratings are first normalized by
* subtracting a baseline score (usually a mean).
*
* @param userMapping The index mapping of user IDs to column numbers.
* @param itemMapping The index mapping of item IDs to row numbers.
* @return A matrix storing the <i>normalized</i> user ratings.
*/
private RealMatrix createRatingMatrix(IdIndexMapping userMapping, IdIndexMapping itemMapping) {
final int nusers = userMapping.size();
final int nitems = itemMapping.size();
// Create a matrix with users on rows and items on columns
logger.info("creating {} by {} rating matrix", nusers, nitems);
RealMatrix matrix = MatrixUtils.createRealMatrix(nusers, nitems);
// populate it with data
Cursor<UserHistory<Event>> users = userEventDAO.streamEventsByUser();
try {
for (UserHistory<Event> user : users) {
// Get the row number for this user
int u = userMapping.getIndex(user.getUserId());
MutableSparseVector ratings = Ratings.userRatingVector(user.filter(Rating.class));
MutableSparseVector baselines = MutableSparseVector.create(ratings.keySet());
baselineScorer.score(user.getUserId(), baselines);
// TODO Populate this user's row with their ratings, minus the baseline scores
for (VectorEntry entry : ratings.fast(State.SET)) {
long itemid = entry.getKey();
int i = itemMapping.getIndex(itemid);
double rating = entry.getValue();
double baseline = baselines.get(itemid);
matrix.setEntry(u, i, rating - baseline);
}
}
} finally {
users.close();
}
return matrix;
}
private List<Integer> findZeroColumns(RealMatrix base) {
List<Integer> indices = new ArrayList<>();
for (int i = 0; i < base.getColumnDimension(); i++) {
if (base.getColumnVector(i).getNorm() == 0) indices.add(i);
}
return indices;
}
private static RealMatrix S2(double t) {
double Omega = 73.6667 + 0.013958 * (Times.MJD(t) + 3242) / 365.25;
double theta0 =
Math.atan(
Math.toRadians(
Math.cos(Math.toRadians(7.25)) * Math.tan(Math.toRadians(lambda0(t) - Omega))));
double angle1 = (lambda0(t) - Omega);
angle1 = angle1 % 360.0;
theta0 = theta0 % 360.0;
if (angle1 < 0.0) angle1 += 360.0;
if (theta0 < 0.0) theta0 += 360.0;
if (angle1 < 180.0) {
if (theta0 < 180.0) theta0 += 180.0;
}
if (angle1 > 180.0) {
if (theta0 > 180.0) theta0 -= 180.0;
}
RealMatrix A = RotationMatrix(theta0, new Vector3D(0, 0, 1));
RealMatrix B = RotationMatrix(7.25, new Vector3D(1, 0, 0));
RealMatrix C = RotationMatrix(Omega, new Vector3D(0, 0, 1));
RealMatrix S2 = A.preMultiply(B).preMultiply(C);
return S2;
}
private float derivEfAlpha(int i) {
float res = 0.0f, xIn, yIn, xA, yA, tmp;
for (int j = 0; j < k; j++) {
xIn = (float) inputFeatPts.getEntry(j, 0);
yIn = (float) inputFeatPts.getEntry(j, 1);
xA = (float) averageFeatPts.getEntry(j, 0);
yA = (float) averageFeatPts.getEntry(j, 1);
tmp = 0.0f;
for (int a = 0; a < 60; a++) {
// tmp += alpha[a] * (subFSV[a][j][0] + subFSV[a][j][2]);
tmp +=
alpha[a]
* (s[(landmarks83Index[j] - 1) * 3][a] + s[(landmarks83Index[j] - 1) * 3 + 2][a]);
}
res +=
-2.0f
* ((xIn + yIn) - (xA + yA) - tmp)
* alpha[i]
// * (subFSV[i][j][0] + subFSV[i][j][2]);
* (s[(landmarks83Index[j] - 1) * 3][i] + s[(landmarks83Index[j] - 1) * 3 + 2][i]);
}
return res;
}
public WeightedLeastSquaresMethod(RealMatrix R, int nFactors, RotationMethod rotationMethod) {
this.nVariables = R.getColumnDimension();
this.nParam = nVariables;
this.nFactors = nFactors;
this.rotationMethod = rotationMethod;
this.R = R;
this.R2 = R.copy();
}
/**
* Throws MathIllegalArgumentException if the matrix does not have at least two columns and two
* rows.
*
* @param matrix matrix to check for sufficiency
* @throws MathIllegalArgumentException if there is insufficient data
*/
private void checkSufficientData(final RealMatrix matrix) {
int nRows = matrix.getRowDimension();
int nCols = matrix.getColumnDimension();
if (nRows < 2 || nCols < 2) {
throw new MathIllegalArgumentException(
LocalizedFormats.INSUFFICIENT_ROWS_AND_COLUMNS, nRows, nCols);
}
}
public double sumMatrix(RealMatrix matrix) {
double sum = 0.0;
for (int i = 0; i < matrix.getRowDimension(); i++) {
for (int j = 0; j < matrix.getColumnDimension(); j++) {
sum += matrix.getEntry(i, j);
}
}
return sum;
}
/////////////////////////////////// Equation 18 ////////////////////////////////////////////////
private float computeEf() {
float res = 0.0f;
for (int j = 0; j < k; j++) {
res +=
pow(inputFeatPts.getEntry(j, 0) - modelFeatPts.getEntry(j, 0), 2)
+ pow(inputFeatPts.getEntry(j, 1) - modelFeatPts.getEntry(j, 1), 2);
}
return res;
}
public double[][] residuals() {
double[][] resid = new double[nItems][nItems];
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
resid[i][j] = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
}
}
return resid;
}
/*Transformation matrices*/
private static RealMatrix P(double t) {
double t0 = Times.T0(t);
double zA = 0.64062 * t0 + 0.00030 * Math.pow(t0, 2);
double thetaA = 0.55675 * t0 + 0.00012 * Math.pow(t0, 2);
double zetaA = 0.64062 * t0 + 0.00008 * Math.pow(t0, 2);
RealMatrix A = RotationMatrix(zA, new Vector3D(0, 0, 1));
RealMatrix B = RotationMatrix(thetaA, new Vector3D(0, 1, 0));
RealMatrix C = RotationMatrix(zetaA, new Vector3D(0, 0, 1));
return A.preMultiply(B).preMultiply(C);
}
private static RealMatrix T5(double t) {
double latangle = Math.toDegrees(magnetic_latitude(t)) - 90.0;
double lonangle = Math.toDegrees(magnetic_longitude(t));
RealMatrix A =
RotationMatrix(
-latangle, new Vector3D(0, 1, 0)); // latangle in original. Change due to java.
RealMatrix B = RotationMatrix(lonangle, new Vector3D(0, 0, 1));
RealMatrix T5 = A.preMultiply(B);
return T5;
}
@Test
public void testInverseIdentity() {
double tol = 0.00001;
Map<Key, Value> input = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
input.put(new Key("1", "", "1"), new Value("1".getBytes()));
// input.put(new Key("1", "", "2"), new Value("1".getBytes()));
// input.put(new Key("2", "", "1"), new Value("1".getBytes()));
input.put(new Key("2", "", "2"), new Value("1".getBytes()));
RealMatrix matrix = MemMatrixUtil.buildMatrix(input.entrySet().iterator(), 2);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(0, matrix.getEntry(0, 1), tol);
Assert.assertEquals(0, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
matrix = MemMatrixUtil.doInverse(matrix, -1);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(0, matrix.getEntry(0, 1), tol);
Assert.assertEquals(0, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
SortedMap<Key, Value> back =
MemMatrixUtil.matrixToMap(new TreeMap<Key, Value>(TestUtil.COMPARE_KEY_TO_COLQ), matrix);
TestUtil.assertEqualDoubleMap(input, back);
// Assert.assertEquals(1, Double.parseDouble(new String(back.get(new Key("1", "",
// "1")).get())), tol);
// Assert.assertEquals(1, Double.parseDouble(new String(back.get(new Key("2", "",
// "2")).get())), tol);
}
public double meanSquaredResidual() {
double ni = Double.valueOf(nItems).doubleValue();
double temp = 0.0, sum = 0.0;
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
temp = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
sum += temp * temp;
}
}
return sum / (ni * ni);
}
public double[][] squaredResiduals() {
double[][] resid = new double[nItems][nItems];
double temp = 0.0;
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
temp = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
resid[i][j] = temp * temp;
}
}
return resid;
}
public double sumSquaredElements(RealMatrix matrix) {
double sum = 0.0;
double v = 0.0;
for (int i = 0; i < matrix.getRowDimension(); i++) {
for (int j = 0; j < matrix.getColumnDimension(); j++) {
v = matrix.getEntry(i, j);
sum += (v * v);
}
}
return sum;
}
/**
* Returns a matrix of standard errors associated with the estimates in the correlation matrix.
* <br>
* <code>getCorrelationStandardErrors().getEntry(i,j)</code> is the standard error associated with
* <code>getCorrelationMatrix.getEntry(i,j)</code>
*
* <p>The formula used to compute the standard error is <br>
* <code>SE<sub>r</sub> = ((1 - r<sup>2</sup>) / (n - 2))<sup>1/2</sup></code> where <code>r
* </code> is the estimated correlation coefficient and <code>n</code> is the number of
* observations in the source dataset.
*
* <p>To use this method, one of the constructors that supply an input matrix must have been used
* to create this instance.
*
* @return matrix of correlation standard errors
* @throws NullPointerException if this instance was created with no data
*/
public RealMatrix getCorrelationStandardErrors() {
int nVars = correlationMatrix.getColumnDimension();
double[][] out = new double[nVars][nVars];
for (int i = 0; i < nVars; i++) {
for (int j = 0; j < nVars; j++) {
double r = correlationMatrix.getEntry(i, j);
out[i][j] = FastMath.sqrt((1 - r * r) / (nObs - 2));
}
}
return new BlockRealMatrix(out);
}
public static RealMatrix getRealMatrixFromJamaMatrix(Matrix m) {
final int rDim = m.getRowDimension();
final int cDim = m.getColumnDimension();
RealMatrix rm = new Array2DRowRealMatrix(rDim, cDim);
for (int i = 0; i < rDim; i++) {
for (int j = 0; j < cDim; j++) {
rm.setEntry(i, j, m.get(i, j));
}
}
return rm;
}
/**
* @return A matrix containing the row stochastic values of the matrix that contains the
* information about the item categorization transposed, to be used by a {@code
* HIRItemScorer}.
*/
public RealMatrix ColumnStochastic() {
for (int i = 0; i < genreSize; i++) {
RealVector forIter = transposed.getRowVector(i);
double sum = forIter.getL1Norm();
if (sum != 0) {
forIter.mapDivideToSelf(sum);
transposed.setRowVector(i, forIter);
}
}
return transposed;
}
private double prediction(long user, long item) {
double baseline = baselineScorer.score(user, item);
try {
RealMatrix userFeature = model.getUserVector(user);
RealMatrix featureWeights = model.getFeatureWeights();
RealMatrix itemFeature = model.getItemVector(item);
double product =
userFeature.multiply(featureWeights).multiply(itemFeature.transpose()).getEntry(0, 0);
return baseline + product;
} catch (NullPointerException npe) {
return baseline;
}
}
public RealMatrix makeZta(RealMatrix userFeature, RealMatrix articleFeature) {
RealMatrix product = userFeature.multiply(articleFeature.transpose());
double[][] productData = product.getData();
double[] productVector = new double[36];
int count = 0;
for (int row = 0; row < 6; row++) {
for (int col = 0; col < 6; col++) {
productVector[count] = productData[row][col];
count++;
}
}
return MatrixUtils.createColumnRealMatrix(productVector);
}
/**
* Function to perform QR decomposition on a given matrix.
*
* @param in matrix object
* @return array of matrix blocks
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock[] computeQR(MatrixObject in) throws DMLRuntimeException {
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);
// Perform QR decomposition
QRDecomposition qrdecompose = new QRDecomposition(matrixInput);
RealMatrix H = qrdecompose.getH();
RealMatrix R = qrdecompose.getR();
// Read the results into native format
MatrixBlock mbH = DataConverter.convertToMatrixBlock(H.getData());
MatrixBlock mbR = DataConverter.convertToMatrixBlock(R.getData());
return new MatrixBlock[] {mbH, mbR};
}
/**
* Build the SVD model.
*
* @return A singular value decomposition recommender model.
*/
@Override
public SVDModel get() {
// Create index mappings of user and item IDs.
// You can use these to find row and columns in the matrix based on user/item IDs.
IdIndexMapping userMapping = IdIndexMapping.create(userDAO.getUserIds());
logger.debug("indexed {} users", userMapping.size());
IdIndexMapping itemMapping = IdIndexMapping.create(itemDAO.getItemIds());
logger.debug("indexed {} items", itemMapping.size());
// We have to do 2 things:
// First, prepare a matrix containing the rating data.
RealMatrix matrix = createRatingMatrix(userMapping, itemMapping);
// Second, compute its factorization
// All the work is done in the constructor
SingularValueDecomposition svd = new SingularValueDecomposition(matrix);
// Third, truncate the decomposed matrix
// TODO Truncate the matrices and construct the SVD model
RealMatrix userMatrix = svd.getU();
RealMatrix weights = svd.getS();
RealMatrix itemMatrix = svd.getV();
userMatrix = userMatrix.getSubMatrix(0, userMatrix.getRowDimension() - 1, 0, featureCount - 1);
weights = weights.getSubMatrix(0, featureCount - 1, 0, featureCount - 1);
itemMatrix = itemMatrix.getSubMatrix(0, itemMatrix.getRowDimension() - 1, 0, featureCount - 1);
return new SVDModel(userMapping, itemMapping, userMatrix, itemMatrix, weights);
}
/**
* Function to compute Cholesky decomposition of the given input matrix. The input must be a real
* symmetric positive-definite matrix.
*
* @param in commons-math3 Array2DRowRealMatrix
* @return matrix block
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock computeCholesky(Array2DRowRealMatrix in) throws DMLRuntimeException {
if (!in.isSquare())
throw new DMLRuntimeException(
"Input to cholesky() must be square matrix -- given: a "
+ in.getRowDimension()
+ "x"
+ in.getColumnDimension()
+ " matrix.");
CholeskyDecomposition cholesky = new CholeskyDecomposition(in);
RealMatrix rmL = cholesky.getL();
return DataConverter.convertToMatrixBlock(rmL.getData());
}
private static double[] calculateColumnInverseMeans(RealMatrix matrix) {
return IntStream.range(0, matrix.getColumnDimension())
.mapToDouble(
i ->
1.0
/ IntStream.range(0, matrix.getRowDimension())
.mapToDouble(j -> matrix.getEntry(j, i))
.average()
.orElseThrow(
() ->
new IllegalArgumentException(
"cannot calculate a average for column " + i)))
.toArray();
}
