private float derivEfAlpha(int i) {
float res = 0.0f, xIn, yIn, xA, yA, tmp;
for (int j = 0; j < k; j++) {
xIn = (float) inputFeatPts.getEntry(j, 0);
yIn = (float) inputFeatPts.getEntry(j, 1);
xA = (float) averageFeatPts.getEntry(j, 0);
yA = (float) averageFeatPts.getEntry(j, 1);
tmp = 0.0f;
for (int a = 0; a < 60; a++) {
// tmp += alpha[a] * (subFSV[a][j][0] + subFSV[a][j][2]);
tmp +=
alpha[a]
* (s[(landmarks83Index[j] - 1) * 3][a] + s[(landmarks83Index[j] - 1) * 3 + 2][a]);
}
res +=
-2.0f
* ((xIn + yIn) - (xA + yA) - tmp)
* alpha[i]
// * (subFSV[i][j][0] + subFSV[i][j][2]);
* (s[(landmarks83Index[j] - 1) * 3][i] + s[(landmarks83Index[j] - 1) * 3 + 2][i]);
}
return res;
}
@Test
public void testInverseIdentity() {
double tol = 0.00001;
Map<Key, Value> input = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
input.put(new Key("1", "", "1"), new Value("1".getBytes()));
// input.put(new Key("1", "", "2"), new Value("1".getBytes()));
// input.put(new Key("2", "", "1"), new Value("1".getBytes()));
input.put(new Key("2", "", "2"), new Value("1".getBytes()));
RealMatrix matrix = MemMatrixUtil.buildMatrix(input.entrySet().iterator(), 2);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(0, matrix.getEntry(0, 1), tol);
Assert.assertEquals(0, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
matrix = MemMatrixUtil.doInverse(matrix, -1);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(0, matrix.getEntry(0, 1), tol);
Assert.assertEquals(0, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
SortedMap<Key, Value> back =
MemMatrixUtil.matrixToMap(new TreeMap<Key, Value>(TestUtil.COMPARE_KEY_TO_COLQ), matrix);
TestUtil.assertEqualDoubleMap(input, back);
// Assert.assertEquals(1, Double.parseDouble(new String(back.get(new Key("1", "",
// "1")).get())), tol);
// Assert.assertEquals(1, Double.parseDouble(new String(back.get(new Key("2", "",
// "2")).get())), tol);
}
@Test
public void testInverse2x2() {
double tol = 0.001;
Map<Key, Value> input = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
input.put(new Key("1", "", "1"), new Value("4".getBytes()));
input.put(new Key("1", "", "2"), new Value("3".getBytes()));
input.put(new Key("2", "", "1"), new Value("1".getBytes()));
input.put(new Key("2", "", "2"), new Value("1".getBytes()));
Map<Key, Value> expect = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
expect.put(new Key("1", "", "1"), new Value("1 ".getBytes()));
expect.put(new Key("1", "", "2"), new Value("-3".getBytes()));
expect.put(new Key("2", "", "1"), new Value("-1".getBytes()));
expect.put(new Key("2", "", "2"), new Value("4 ".getBytes()));
RealMatrix matrix = MemMatrixUtil.buildMatrix(input.entrySet().iterator(), 2);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(4, matrix.getEntry(0, 0), tol);
Assert.assertEquals(3, matrix.getEntry(0, 1), tol);
Assert.assertEquals(1, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
matrix = MemMatrixUtil.doInverse(matrix, -1);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(-3, matrix.getEntry(0, 1), tol);
Assert.assertEquals(-1, matrix.getEntry(1, 0), tol);
Assert.assertEquals(4, matrix.getEntry(1, 1), tol);
SortedMap<Key, Value> back =
MemMatrixUtil.matrixToMap(new TreeMap<Key, Value>(TestUtil.COMPARE_KEY_TO_COLQ), matrix);
TestUtil.assertEqualDoubleMap(expect, back);
}
public double[][] residuals() {
double[][] resid = new double[nItems][nItems];
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
resid[i][j] = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
}
}
return resid;
}
/////////////////////////////////// Equation 18 ////////////////////////////////////////////////
private float computeEf() {
float res = 0.0f;
for (int j = 0; j < k; j++) {
res +=
pow(inputFeatPts.getEntry(j, 0) - modelFeatPts.getEntry(j, 0), 2)
+ pow(inputFeatPts.getEntry(j, 1) - modelFeatPts.getEntry(j, 1), 2);
}
return res;
}
public double[][] squaredResiduals() {
double[][] resid = new double[nItems][nItems];
double temp = 0.0;
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
temp = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
resid[i][j] = temp * temp;
}
}
return resid;
}
public double meanSquaredResidual() {
double ni = Double.valueOf(nItems).doubleValue();
double temp = 0.0, sum = 0.0;
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
temp = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
sum += temp * temp;
}
}
return sum / (ni * ni);
}
/**
* Gradient
*
* @param x a <code>double[]</code> input vector
* @return
*/
@Override
public double[] gradient(double[] x) {
double[] sqrtPsi = new double[nVariables];
double[] invSqrtPsi = new double[nVariables];
for (int i = 0; i < nVariables; i++) {
x[i] = Math.max(0.005, x[i]); // ensure that no parameters are negative
sqrtPsi[i] = Math.sqrt(x[i]);
invSqrtPsi[i] = 1.0 / Math.sqrt(x[i]);
}
DiagonalMatrix diagPsi = new DiagonalMatrix(x);
DiagonalMatrix diagSqtPsi = new DiagonalMatrix(sqrtPsi);
DiagonalMatrix SC = new DiagonalMatrix(invSqrtPsi);
RealMatrix Sstar = SC.multiply(R2).multiply(SC);
EigenDecomposition E = new EigenDecomposition(Sstar);
RealMatrix L = E.getV().getSubMatrix(0, nVariables - 1, 0, nFactors - 1);
double[] ev = new double[nFactors];
for (int i = 0; i < nFactors; i++) {
ev[i] = Math.sqrt(Math.max(E.getRealEigenvalue(i) - 1, 0));
}
DiagonalMatrix M = new DiagonalMatrix(ev);
RealMatrix LOAD = L.multiply(M);
RealMatrix LL = diagSqtPsi.multiply(LOAD);
RealMatrix G = LL.multiply(LL.transpose()).add(diagPsi).subtract(R2);
double[] gradient = new double[nVariables];
for (int i = 0; i < nVariables; i++) {
gradient[i] = G.getEntry(i, i) / (x[i] * x[i]);
}
return gradient;
}
/**
* Calculates {@code P(D_n < d)} using method described in [1] and doubles (see above).
*
* @param d statistic
* @return the two-sided probability of {@code P(D_n < d)}
* @throws MathArithmeticException if algorithm fails to convert {@code h} to a {@link
* org.apache.commons.math3.fraction.BigFraction} in expressing {@code d} as {@code (k - h) /
* m} for integer {@code k, m} and {@code 0 <= h < 1}.
*/
private double roundedK(double d) throws MathArithmeticException {
final int k = (int) FastMath.ceil(n * d);
final FieldMatrix<BigFraction> HBigFraction = this.createH(d);
final int m = HBigFraction.getRowDimension();
/*
* Here the rounding part comes into play: use
* RealMatrix instead of FieldMatrix<BigFraction>
*/
final RealMatrix H = new Array2DRowRealMatrix(m, m);
for (int i = 0; i < m; ++i) {
for (int j = 0; j < m; ++j) {
H.setEntry(i, j, HBigFraction.getEntry(i, j).doubleValue());
}
}
final RealMatrix Hpower = H.power(n);
double pFrac = Hpower.getEntry(k - 1, k - 1);
for (int i = 1; i <= n; ++i) {
pFrac *= (double) i / (double) n;
}
return pFrac;
}
/**
* Derives a correlation matrix from a covariance matrix.
*
* <p>Uses the formula <br>
* <code>r(X,Y) = cov(X,Y)/s(X)s(Y)</code> where <code>r(·,·)</code> is the
* correlation coefficient and <code>s(·)</code> means standard deviation.
*
* @param covarianceMatrix the covariance matrix
* @return correlation matrix
*/
public RealMatrix covarianceToCorrelation(RealMatrix covarianceMatrix) {
int nVars = covarianceMatrix.getColumnDimension();
RealMatrix outMatrix = new BlockRealMatrix(nVars, nVars);
for (int i = 0; i < nVars; i++) {
double sigma = FastMath.sqrt(covarianceMatrix.getEntry(i, i));
outMatrix.setEntry(i, i, 1d);
for (int j = 0; j < i; j++) {
double entry =
covarianceMatrix.getEntry(i, j)
/ (sigma * FastMath.sqrt(covarianceMatrix.getEntry(j, j)));
outMatrix.setEntry(i, j, entry);
outMatrix.setEntry(j, i, entry);
}
}
return outMatrix;
}
public double sumMatrix(RealMatrix matrix) {
double sum = 0.0;
for (int i = 0; i < matrix.getRowDimension(); i++) {
for (int j = 0; j < matrix.getColumnDimension(); j++) {
sum += matrix.getEntry(i, j);
}
}
return sum;
}
// Berechnung des Zustandswertes (Gebaeudezustand) des Interpolationspunktes
public double calculateIntValue() {
double value = 0;
for (int i = 0; i < numOfPoints; i++) {
value = value + weightVector.getEntry(i, 0) * points.get(i).data;
}
return value;
}
public double valueAt(double[] param) {
double[] sdInv = new double[nVariables];
for (int i = 0; i < nVariables; i++) {
R.setEntry(i, i, 1.0 - param[i]);
sdInv[i] = 1.0 / Sinv.getEntry(i, i);
}
DiagonalMatrix diagSdInv = new DiagonalMatrix(sdInv);
EigenDecomposition eigen = new EigenDecomposition(R);
RealMatrix eigenVectors = eigen.getV().getSubMatrix(0, nVariables - 1, 0, nFactors - 1);
double[] ev = new double[nFactors];
for (int i = 0; i < nFactors; i++) {
ev[i] = Math.sqrt(eigen.getRealEigenvalue(i));
}
DiagonalMatrix evMatrix =
new DiagonalMatrix(
ev); // USE Apache version of Diagonal matrix when upgrade to version 3.2
RealMatrix LAMBDA = eigenVectors.multiply(evMatrix);
RealMatrix SIGMA = (LAMBDA.multiply(LAMBDA.transpose()));
double value = 0.0;
RealMatrix DIF = R.subtract(SIGMA);
for (int i = 0; i < DIF.getRowDimension(); i++) {
for (int j = 0; j < DIF.getColumnDimension(); j++) {
value = DIF.getEntry(i, j);
DIF.setEntry(i, j, Math.pow(value, 2));
}
}
RealMatrix RESID = diagSdInv.multiply(DIF).multiply(diagSdInv);
double sum = 0.0;
for (int i = 0; i < RESID.getRowDimension(); i++) {
for (int j = 0; j < RESID.getColumnDimension(); j++) {
sum += RESID.getEntry(i, j);
}
}
return sum;
}
/**
* Returns a matrix of standard errors associated with the estimates in the correlation matrix.
* <br>
* <code>getCorrelationStandardErrors().getEntry(i,j)</code> is the standard error associated with
* <code>getCorrelationMatrix.getEntry(i,j)</code>
*
* <p>The formula used to compute the standard error is <br>
* <code>SE<sub>r</sub> = ((1 - r<sup>2</sup>) / (n - 2))<sup>1/2</sup></code> where <code>r
* </code> is the estimated correlation coefficient and <code>n</code> is the number of
* observations in the source dataset.
*
* <p>To use this method, one of the constructors that supply an input matrix must have been used
* to create this instance.
*
* @return matrix of correlation standard errors
* @throws NullPointerException if this instance was created with no data
*/
public RealMatrix getCorrelationStandardErrors() {
int nVars = correlationMatrix.getColumnDimension();
double[][] out = new double[nVars][nVars];
for (int i = 0; i < nVars; i++) {
for (int j = 0; j < nVars; j++) {
double r = correlationMatrix.getEntry(i, j);
out[i][j] = FastMath.sqrt((1 - r * r) / (nObs - 2));
}
}
return new BlockRealMatrix(out);
}
public double sumSquaredElements(RealMatrix matrix) {
double sum = 0.0;
double v = 0.0;
for (int i = 0; i < matrix.getRowDimension(); i++) {
for (int j = 0; j < matrix.getColumnDimension(); j++) {
v = matrix.getEntry(i, j);
sum += (v * v);
}
}
return sum;
}
@Test
public void testTransposeTimesSelf() {
Map<Integer, float[]> a = new HashMap<>();
a.put(-1, new float[] {1.3f, -2.0f, 3.0f});
a.put(1, new float[] {2.0f, 0.0f, 5.0f});
a.put(3, new float[] {0.0f, -1.5f, 5.5f});
RealMatrix ata = VectorMath.transposeTimesSelf(a.values());
RealMatrix expected =
new Array2DRowRealMatrix(
new double[][] {
{5.69, -2.6, 13.9},
{-2.6, 6.25, -14.25},
{13.9, -14.25, 64.25}
});
for (int row = 0; row < 3; row++) {
for (int col = 0; col < 3; col++) {
assertEquals(expected.getEntry(row, col), ata.getEntry(row, col), FLOAT_EPSILON);
}
}
}
// Makes and scales the matrices V, D, and VT (to avoid ugly decimals)
private void makeVDVT(EigenDecomp ed) {
V = ed.getV();
D = ed.getD();
VT = ed.getVT();
double ref = 0;
for (int i = 0; i < V.getRowDimension(); i++) {
ref = 0;
for (int j = 0; j < V.getColumnDimension(); j++) {
if (V.getEntry(j, i) != 0 && ref == 0) {
ref = V.getEntry(j, i);
}
if (ref != 0) {
V.setEntry(j, i, V.getEntry(j, i) / Math.abs(ref));
}
}
}
for (int i = 0; i < VT.getRowDimension(); i++) {
ref = 0;
for (int j = 0; j < VT.getColumnDimension(); j++) {
if (VT.getEntry(j, i) != 0 && ref == 0) {
ref = VT.getEntry(j, i);
}
if (ref != 0) {
VT.setEntry(j, i, VT.getEntry(j, i) / Math.abs(ref));
}
}
}
}
@Override
public void train(List<Instance> instances) {
// ------------------------ initialize rows and columns ---------------------
int rows = instances.size();
int columns = 0;
// get max columns
for (Instance i : instances) {
int localColumns = Collections.max(i.getFeatureVector().getFeatureMap().keySet());
if (localColumns > columns) columns = localColumns;
}
// ------------------------ initialize alpha vector -----------------------
alpha = new ArrayRealVector(rows, 0);
// ------------------------ initialize base X and Y for use --------------------------
double[][] X = new double[rows][columns];
double[] Y = new double[rows];
for (int i = 0; i < rows; i++) {
Y[i] = ((ClassificationLabel) instances.get(i).getLabel()).getLabelValue();
for (int j = 0; j < columns; j++) {
X[i][j] = instances.get(i).getFeatureVector().get(j + 1);
}
}
// ---------------------- gram matrix -------------------
matrixX = new Array2DRowRealMatrix(X);
RealMatrix gram = new Array2DRowRealMatrix(rows, rows);
for (int i = 0; i < rows; i++) {
for (int j = 0; j < rows; j++) {
gram.setEntry(i, j, kernelFunction(matrixX.getRowVector(i), matrixX.getRowVector(j)));
}
}
// ---------------------- gradient ascent --------------------------
Sigmoid g = new Sigmoid(); // helper function
System.out.println("Training start...");
System.out.println(
"Learning rate: " + _learning_rate + " Training times: " + _training_iterations);
for (int idx = 0; idx < _training_iterations; idx++) {
System.out.println("Training iteration: " + (idx + 1));
for (int k = 0; k < rows; k++) {
double gradient_ascent = 0.0;
RealVector alpha_gram = gram.operate(alpha);
for (int i = 0; i < rows; i++) {
double lambda = alpha_gram.getEntry(i);
double kernel = gram.getEntry(i, k);
gradient_ascent =
gradient_ascent
+ Y[i] * g.value(-lambda) * kernel
+ (1 - Y[i]) * g.value(lambda) * (-kernel);
}
alpha.setEntry(k, alpha.getEntry(k) + _learning_rate * gradient_ascent);
}
}
System.out.println("Training done!");
}
private RealMatrix normalizeData(RealMatrix matrix, UserProfileEigenModel model) {
RealMatrix normalizedData =
new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension());
if (LOG.isDebugEnabled()) LOG.debug("model statistics size: " + model.statistics().length);
for (int i = 0; i < matrix.getRowDimension(); i++) {
for (int j = 0; j < matrix.getColumnDimension(); j++) {
double value =
(matrix.getEntry(i, j) - model.statistics()[j].getMean())
/ model.statistics()[j].getStddev();
normalizedData.setEntry(i, j, value);
}
}
return normalizedData;
}
private static double[] calculateColumnInverseMeans(RealMatrix matrix) {
return IntStream.range(0, matrix.getColumnDimension())
.mapToDouble(
i ->
1.0
/ IntStream.range(0, matrix.getRowDimension())
.mapToDouble(j -> matrix.getEntry(j, i))
.average()
.orElseThrow(
() ->
new IllegalArgumentException(
"cannot calculate a average for column " + i)))
.toArray();
}
public double[] getStartValues() {
double[] start = new double[nVariables];
if (nFactors == 1) {
for (int i = 0; i < nVariables; i++) {
start[i] = 0.5;
}
} else {
for (int i = 0; i < nVariables; i++) {
start[i] = Math.min(1.0 / Sinv.getEntry(i, i), 1.0);
}
}
return start;
}
private RealMatrix removeZeroColumns(RealMatrix base, List<Integer> zeroColumns) {
int adjustedDim = base.getRowDimension() - zeroColumns.size();
if (adjustedDim == 0) return base;
RealMatrix adjusted = new Array2DRowRealMatrix(adjustedDim, adjustedDim);
int i = 0, j = 0;
for (int basei = 0; basei < base.getRowDimension(); basei++) {
if (zeroColumns.contains(basei)) continue;
for (int basej = 0; basej < base.getColumnDimension(); basej++) {
if (zeroColumns.contains(basej)) continue;
adjusted.setEntry(i, j++, base.getEntry(basei, basej));
}
i++;
j = 0;
}
return adjusted;
}
/**
* Returns a matrix of p-values associated with the (two-sided) null hypothesis that the
* corresponding correlation coefficient is zero.
*
* <p><code>getCorrelationPValues().getEntry(i,j)</code> is the probability that a random variable
* distributed as <code>t<sub>n-2</sub></code> takes a value with absolute value greater than or
* equal to <br>
* <code>|r|((n - 2) / (1 - r<sup>2</sup>))<sup>1/2</sup></code>
*
* <p>The values in the matrix are sometimes referred to as the <i>significance</i> of the
* corresponding correlation coefficients.
*
* <p>To use this method, one of the constructors that supply an input matrix must have been used
* to create this instance.
*
* @return matrix of p-values
* @throws org.apache.commons.math3.exception.MaxCountExceededException if an error occurs
* estimating probabilities
* @throws NullPointerException if this instance was created with no data
*/
public RealMatrix getCorrelationPValues() {
TDistribution tDistribution = new TDistribution(nObs - 2);
int nVars = correlationMatrix.getColumnDimension();
double[][] out = new double[nVars][nVars];
for (int i = 0; i < nVars; i++) {
for (int j = 0; j < nVars; j++) {
if (i == j) {
out[i][j] = 0d;
} else {
double r = correlationMatrix.getEntry(i, j);
double t = FastMath.abs(r * FastMath.sqrt((nObs - 2) / (1 - r * r)));
out[i][j] = 2 * tDistribution.cumulativeProbability(-t);
}
}
}
return new BlockRealMatrix(out);
}
private static RealMatrix T6(double t) {
double theta, thetaN, magThetaD;
// Earth-Sun vector in GSE
Coordinate GSE_ES = new Coordinate("GSE", 1, 0, 0, t);
// Convert GSE --> GEI
Coordinate GEI = toGEI(GSE_ES);
RealMatrix GEI_ES = CoordinateToMatrix(GEI);
thetaN =
GEI_ES.getEntry(0, 0)
* (-0.032 + GEI_ES.getEntry(1, 0) * -0.112 + GEI_ES.getEntry(2, 0) * -0.048);
Vector3D ThetaD =
new Vector3D(
-0.424 * GEI_ES.getEntry(2, 0) - 0.897 * GEI_ES.getEntry(1, 0),
0.897 * GEI_ES.getEntry(0, 0) - 0.1217 * GEI_ES.getEntry(2, 0),
0.1217 * GEI_ES.getEntry(1, 0) - -0.424 * GEI_ES.getEntry(0, 0));
magThetaD = ThetaD.getNorm();
theta = Math.asin(thetaN / magThetaD);
RealMatrix T6 = RotationMatrix(Math.toDegrees(theta), new Vector3D(1, 0, 0));
return T6.transpose(); // unknown why transpose is needed to match previous results
}
@Override
public List<MLCallbackResult> detect(
final String user,
final String algorithm,
UserActivityAggModel userActivity,
UserProfileEigenModel aModel) {
RealMatrix inputData = userActivity.matrix();
LOG.warn(
"EigenBasedAnomalyDetection predictAnomaly called with dimension: "
+ inputData.getRowDimension()
+ "x"
+ inputData.getColumnDimension());
if (aModel == null) {
LOG.warn(
"nothing to do as the input model does not have required values, returning from evaluating this algorithm..");
return null;
}
List<MLCallbackResult> mlCallbackResults = new ArrayList<MLCallbackResult>();
RealMatrix normalizedMat = normalizeData(inputData, aModel);
UserCommandStatistics[] listStats = aModel.statistics();
int colWithHighVariant = 0;
for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
if (listStats[j].isLowVariant() == false) {
colWithHighVariant++;
}
}
final Map<String, String> context =
new HashMap<String, String>() {
{
put(UserProfileConstants.USER_TAG, user);
put(UserProfileConstants.ALGORITHM_TAG, algorithm);
}
};
Map<Integer, String> lineNoWithVariantBasedAnomalyDetection = new HashMap<Integer, String>();
for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
MLCallbackResult aResult = new MLCallbackResult();
aResult.setContext(context);
for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
// LOG.info("mean for j=" + j + " is:" + listStats[j].getMean());
// LOG.info("stddev for j=" + j + " is:" + listStats[j].getStddev());
if (listStats[j].isLowVariant() == true) {
// LOG.info(listOfCmds[j] + " is low variant");
if (normalizedMat.getEntry(i, j) > listStats[j].getMean()) {
lineNoWithVariantBasedAnomalyDetection.put(i, "lowVariantAnomaly");
aResult.setAnomaly(true);
aResult.setTimestamp(userActivity.timestamp());
aResult.setFeature(listStats[j].getCommandName());
aResult.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
List<String> datapoints = new ArrayList<String>();
double[] rowVals = inputData.getRow(i);
for (double rowVal : rowVals) datapoints.add(rowVal + "");
aResult.setDatapoints(datapoints);
aResult.setId(user);
mlCallbackResults.add(aResult);
} else {
aResult.setAnomaly(false);
aResult.setTimestamp(userActivity.timestamp());
mlCallbackResults.add(aResult);
}
}
}
// return results;
}
// LOG.info("results size here: " + results.length);
// LOG.info("col with high variant: " + colWithHighVariant);
RealMatrix finalMatWithoutLowVariantFeatures =
new Array2DRowRealMatrix(normalizedMat.getRowDimension(), colWithHighVariant);
// LOG.info("size of final test data: " + finalMatWithoutLowVariantFeatures.getRowDimension()
// +"x"+ finalMatWithoutLowVariantFeatures.getColumnDimension());
int finalMatrixRow = 0;
int finalMatrixCol = 0;
for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
if (listStats[j].isLowVariant() == false) {
finalMatWithoutLowVariantFeatures.setEntry(
finalMatrixRow, finalMatrixCol, normalizedMat.getEntry(i, j));
finalMatrixCol++;
}
}
finalMatrixCol = 0;
finalMatrixRow++;
}
RealVector[] pcs = aModel.principalComponents();
// LOG.info("pc size: " + pcs.getRowDimension() +"x" + pcs.getColumnDimension());
RealMatrix finalInputMatTranspose = finalMatWithoutLowVariantFeatures.transpose();
for (int i = 0; i < finalMatWithoutLowVariantFeatures.getRowDimension(); i++) {
if (lineNoWithVariantBasedAnomalyDetection.get(i) == null) {
MLCallbackResult result = new MLCallbackResult();
result.setContext(context);
for (int sz = 0; sz < pcs.length; sz++) {
double[] pc1 = pcs[sz].toArray();
RealMatrix pc1Mat = new Array2DRowRealMatrix(pc1);
RealMatrix transposePC1Mat = pc1Mat.transpose();
RealMatrix testData =
pc1Mat.multiply(transposePC1Mat).multiply(finalInputMatTranspose.getColumnMatrix(i));
// LOG.info("testData size: " + testData.getRowDimension() + "x" +
// testData.getColumnDimension());
RealMatrix testDataTranspose = testData.transpose();
// LOG.info("testData transpose size: " + testDataTranspose.getRowDimension() + "x" +
// testDataTranspose.getColumnDimension());
RealVector iRowVector = testDataTranspose.getRowVector(0);
// RealVector pc1Vector = transposePC1Mat.getRowVector(sz);
RealVector pc1Vector = transposePC1Mat.getRowVector(0);
double distanceiRowAndPC1 = iRowVector.getDistance(pc1Vector);
// LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " +
// model.getMaxL2Norm().getEntry(sz));
// LOG.info("model.getMaxL2Norm().getEntry(sz):" + model.getMaxL2Norm().getEntry(sz));
if (distanceiRowAndPC1 > aModel.maximumL2Norm().getEntry(sz)) {
// LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " +
// model.getMaxL2Norm().getEntry(sz));
result.setAnomaly(true);
result.setFeature(aModel.statistics()[sz].getCommandName());
result.setTimestamp(System.currentTimeMillis());
result.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
List<String> datapoints = new ArrayList<String>();
double[] rowVals = inputData.getRow(i);
for (double rowVal : rowVals) datapoints.add(rowVal + "");
result.setDatapoints(datapoints);
result.setId(user);
}
}
mlCallbackResults.add(result);
}
}
return mlCallbackResults;
}
public double rateFor(final int currentPage, final int pageIndex) {
return matrix.getEntry(currentPage, pageIndex);
}
private void computeFactorLoadings(double[] x) {
uniqueness = x;
communality = new double[nVariables];
for (int i = 0; i < nVariables; i++) {
R.setEntry(i, i, 1.0 - x[i]);
}
EigenDecomposition E = new EigenDecomposition(R);
RealMatrix L = E.getV().getSubMatrix(0, nVariables - 1, 0, nFactors - 1);
double[] ev = new double[nFactors];
for (int i = 0; i < nFactors; i++) {
ev[i] = Math.sqrt(E.getRealEigenvalue(i));
}
DiagonalMatrix M = new DiagonalMatrix(ev);
RealMatrix LOAD = L.multiply(M);
// rotate factor loadings
if (rotationMethod != RotationMethod.NONE) {
GPArotation gpa = new GPArotation();
RotationResults results = gpa.rotate(LOAD, rotationMethod);
LOAD = results.getFactorLoadings();
}
Sum[] colSums = new Sum[nFactors];
Sum[] colSumsSquares = new Sum[nFactors];
for (int j = 0; j < nFactors; j++) {
colSums[j] = new Sum();
colSumsSquares[j] = new Sum();
}
factorLoading = new double[nVariables][nFactors];
for (int i = 0; i < nVariables; i++) {
for (int j = 0; j < nFactors; j++) {
factorLoading[i][j] = LOAD.getEntry(i, j);
colSums[j].increment(factorLoading[i][j]);
colSumsSquares[j].increment(Math.pow(factorLoading[i][j], 2));
communality[i] += Math.pow(factorLoading[i][j], 2);
}
}
// check sign of factor
double sign = 1.0;
for (int i = 0; i < nVariables; i++) {
for (int j = 0; j < nFactors; j++) {
if (colSums[j].getResult() < 0) {
sign = -1.0;
} else {
sign = 1.0;
}
factorLoading[i][j] = factorLoading[i][j] * sign;
}
}
double totSumOfSquares = 0.0;
sumsOfSquares = new double[nFactors];
proportionOfExplainedVariance = new double[nFactors];
proportionOfVariance = new double[nFactors];
for (int j = 0; j < nFactors; j++) {
sumsOfSquares[j] = colSumsSquares[j].getResult();
totSumOfSquares += sumsOfSquares[j];
}
for (int j = 0; j < nFactors; j++) {
proportionOfExplainedVariance[j] = sumsOfSquares[j] / totSumOfSquares;
proportionOfVariance[j] = sumsOfSquares[j] / nVariables;
}
}