// Makes and scales the matrices V, D, and VT (to avoid ugly decimals)
private void makeVDVT(EigenDecomp ed) {
V = ed.getV();
D = ed.getD();
VT = ed.getVT();
double ref = 0;
for (int i = 0; i < V.getRowDimension(); i++) {
ref = 0;
for (int j = 0; j < V.getColumnDimension(); j++) {
if (V.getEntry(j, i) != 0 && ref == 0) {
ref = V.getEntry(j, i);
}
if (ref != 0) {
V.setEntry(j, i, V.getEntry(j, i) / Math.abs(ref));
}
}
}
for (int i = 0; i < VT.getRowDimension(); i++) {
ref = 0;
for (int j = 0; j < VT.getColumnDimension(); j++) {
if (VT.getEntry(j, i) != 0 && ref == 0) {
ref = VT.getEntry(j, i);
}
if (ref != 0) {
VT.setEntry(j, i, VT.getEntry(j, i) / Math.abs(ref));
}
}
}
}
/**
* Build the SVD model.
*
* @return A singular value decomposition recommender model.
*/
@Override
public SVDModel get() {
// Create index mappings of user and item IDs.
// You can use these to find row and columns in the matrix based on user/item IDs.
IdIndexMapping userMapping = IdIndexMapping.create(userDAO.getUserIds());
logger.debug("indexed {} users", userMapping.size());
IdIndexMapping itemMapping = IdIndexMapping.create(itemDAO.getItemIds());
logger.debug("indexed {} items", itemMapping.size());
// We have to do 2 things:
// First, prepare a matrix containing the rating data.
RealMatrix matrix = createRatingMatrix(userMapping, itemMapping);
// Second, compute its factorization
// All the work is done in the constructor
SingularValueDecomposition svd = new SingularValueDecomposition(matrix);
// Third, truncate the decomposed matrix
// TODO Truncate the matrices and construct the SVD model
RealMatrix userMatrix = svd.getU();
RealMatrix weights = svd.getS();
RealMatrix itemMatrix = svd.getV();
userMatrix = userMatrix.getSubMatrix(0, userMatrix.getRowDimension() - 1, 0, featureCount - 1);
weights = weights.getSubMatrix(0, featureCount - 1, 0, featureCount - 1);
itemMatrix = itemMatrix.getSubMatrix(0, itemMatrix.getRowDimension() - 1, 0, featureCount - 1);
return new SVDModel(userMapping, itemMapping, userMatrix, itemMatrix, weights);
}
@Test
public void testInverseIdentity() {
double tol = 0.00001;
Map<Key, Value> input = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
input.put(new Key("1", "", "1"), new Value("1".getBytes()));
// input.put(new Key("1", "", "2"), new Value("1".getBytes()));
// input.put(new Key("2", "", "1"), new Value("1".getBytes()));
input.put(new Key("2", "", "2"), new Value("1".getBytes()));
RealMatrix matrix = MemMatrixUtil.buildMatrix(input.entrySet().iterator(), 2);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(0, matrix.getEntry(0, 1), tol);
Assert.assertEquals(0, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
matrix = MemMatrixUtil.doInverse(matrix, -1);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(0, matrix.getEntry(0, 1), tol);
Assert.assertEquals(0, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
SortedMap<Key, Value> back =
MemMatrixUtil.matrixToMap(new TreeMap<Key, Value>(TestUtil.COMPARE_KEY_TO_COLQ), matrix);
TestUtil.assertEqualDoubleMap(input, back);
// Assert.assertEquals(1, Double.parseDouble(new String(back.get(new Key("1", "",
// "1")).get())), tol);
// Assert.assertEquals(1, Double.parseDouble(new String(back.get(new Key("2", "",
// "2")).get())), tol);
}
/**
* @param weight Weight matrix.
* @throws NonSquareMatrixException if the argument is not a square matrix.
*/
public Weight(RealMatrix weight) {
if (weight.getColumnDimension() != weight.getRowDimension()) {
throw new NonSquareMatrixException(weight.getColumnDimension(), weight.getRowDimension());
}
weightMatrix = weight.copy();
}
@Test
public void testInverse2x2() {
double tol = 0.001;
Map<Key, Value> input = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
input.put(new Key("1", "", "1"), new Value("4".getBytes()));
input.put(new Key("1", "", "2"), new Value("3".getBytes()));
input.put(new Key("2", "", "1"), new Value("1".getBytes()));
input.put(new Key("2", "", "2"), new Value("1".getBytes()));
Map<Key, Value> expect = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
expect.put(new Key("1", "", "1"), new Value("1 ".getBytes()));
expect.put(new Key("1", "", "2"), new Value("-3".getBytes()));
expect.put(new Key("2", "", "1"), new Value("-1".getBytes()));
expect.put(new Key("2", "", "2"), new Value("4 ".getBytes()));
RealMatrix matrix = MemMatrixUtil.buildMatrix(input.entrySet().iterator(), 2);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(4, matrix.getEntry(0, 0), tol);
Assert.assertEquals(3, matrix.getEntry(0, 1), tol);
Assert.assertEquals(1, matrix.getEntry(1, 0), tol);
Assert.assertEquals(1, matrix.getEntry(1, 1), tol);
matrix = MemMatrixUtil.doInverse(matrix, -1);
Assert.assertEquals(2, matrix.getRowDimension());
Assert.assertEquals(2, matrix.getColumnDimension());
Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
Assert.assertEquals(-3, matrix.getEntry(0, 1), tol);
Assert.assertEquals(-1, matrix.getEntry(1, 0), tol);
Assert.assertEquals(4, matrix.getEntry(1, 1), tol);
SortedMap<Key, Value> back =
MemMatrixUtil.matrixToMap(new TreeMap<Key, Value>(TestUtil.COMPARE_KEY_TO_COLQ), matrix);
TestUtil.assertEqualDoubleMap(expect, back);
}
public double computeSimilarity(RealMatrix sourceDoc, RealMatrix targetDoc) {
if (sourceDoc.getRowDimension() != targetDoc.getRowDimension()
|| sourceDoc.getColumnDimension() != targetDoc.getColumnDimension()
|| sourceDoc.getColumnDimension() != 1) {
throw new IllegalArgumentException(
"Matrices are not column matrices or not of the same size");
}
double[] source = sourceDoc.getColumn(0);
double[] target = targetDoc.getColumn(0);
double dotProduct = dot(source, target);
double distance = norm(source) * norm(target);
return dotProduct / distance;
}
private RealMatrix normalizeData(RealMatrix matrix, UserProfileEigenModel model) {
RealMatrix normalizedData =
new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension());
if (LOG.isDebugEnabled()) LOG.debug("model statistics size: " + model.statistics().length);
for (int i = 0; i < matrix.getRowDimension(); i++) {
for (int j = 0; j < matrix.getColumnDimension(); j++) {
double value =
(matrix.getEntry(i, j) - model.statistics()[j].getMean())
/ model.statistics()[j].getStddev();
normalizedData.setEntry(i, j, value);
}
}
return normalizedData;
}
/**
* Applies rank transform to each of the columns of <code>matrix</code> using the current <code>
* rankingAlgorithm</code>
*
* @param matrix matrix to transform
* @return a rank-transformed matrix
*/
private RealMatrix rankTransform(final RealMatrix matrix) {
RealMatrix transformed = null;
if (rankingAlgorithm instanceof NaturalRanking
&& ((NaturalRanking) rankingAlgorithm).getNanStrategy() == NaNStrategy.REMOVED) {
final Set<Integer> nanPositions = new HashSet<Integer>();
for (int i = 0; i < matrix.getColumnDimension(); i++) {
nanPositions.addAll(getNaNPositions(matrix.getColumn(i)));
}
// if we have found NaN values, we have to update the matrix size
if (!nanPositions.isEmpty()) {
transformed =
new BlockRealMatrix(
matrix.getRowDimension() - nanPositions.size(), matrix.getColumnDimension());
for (int i = 0; i < transformed.getColumnDimension(); i++) {
transformed.setColumn(i, removeValues(matrix.getColumn(i), nanPositions));
}
}
}
if (transformed == null) {
transformed = matrix.copy();
}
for (int i = 0; i < transformed.getColumnDimension(); i++) {
transformed.setColumn(i, rankingAlgorithm.rank(transformed.getColumn(i)));
}
return transformed;
}
private RealMatrix removeZeroColumns(RealMatrix base, List<Integer> zeroColumns) {
int adjustedDim = base.getRowDimension() - zeroColumns.size();
if (adjustedDim == 0) return base;
RealMatrix adjusted = new Array2DRowRealMatrix(adjustedDim, adjustedDim);
int i = 0, j = 0;
for (int basei = 0; basei < base.getRowDimension(); basei++) {
if (zeroColumns.contains(basei)) continue;
for (int basej = 0; basej < base.getColumnDimension(); basej++) {
if (zeroColumns.contains(basej)) continue;
adjusted.setEntry(i, j++, base.getEntry(basei, basej));
}
i++;
j = 0;
}
return adjusted;
}
public static RealMatrix stochasticSubmatrix(RealMatrix data, int batch_size, Random rng) {
// assume all data has the size number_samples by number_features
int num_samples = data.getRowDimension();
int num_features = data.getColumnDimension();
int batch_num = num_samples / batch_size + 1;
// randomly generate a batch index
int batch_index = rng.nextInt(batch_num);
List<Integer> rowIndex_tmp = new ArrayList<Integer>();
for (int i = 0; i < batch_size; i++) {
if (batch_size * batch_index + i >= num_samples) {
break;
} else {
rowIndex_tmp.add(batch_size * batch_index + i);
}
}
int[] rowIndex = TypeConvert.ArrayTointv(rowIndex_tmp);
// System.out.println(rowIndex_tmp);
int[] columnIndex = new int[num_features];
for (int j = 0; j < num_features; j++) {
columnIndex[j] = j;
}
// System.out.println(batch_index);
// return null;
return data.getSubMatrix(rowIndex, columnIndex);
}
/**
* Throws MathIllegalArgumentException if the matrix does not have at least two columns and two
* rows.
*
* @param matrix matrix to check for sufficiency
* @throws MathIllegalArgumentException if there is insufficient data
*/
private void checkSufficientData(final RealMatrix matrix) {
int nRows = matrix.getRowDimension();
int nCols = matrix.getColumnDimension();
if (nRows < 2 || nCols < 2) {
throw new MathIllegalArgumentException(
LocalizedFormats.INSUFFICIENT_ROWS_AND_COLUMNS, nRows, nCols);
}
}
public double sumMatrix(RealMatrix matrix) {
double sum = 0.0;
for (int i = 0; i < matrix.getRowDimension(); i++) {
for (int j = 0; j < matrix.getColumnDimension(); j++) {
sum += matrix.getEntry(i, j);
}
}
return sum;
}
public double[][] residuals() {
double[][] resid = new double[nItems][nItems];
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
resid[i][j] = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
}
}
return resid;
}
public double valueAt(double[] param) {
double[] sdInv = new double[nVariables];
for (int i = 0; i < nVariables; i++) {
R.setEntry(i, i, 1.0 - param[i]);
sdInv[i] = 1.0 / Sinv.getEntry(i, i);
}
DiagonalMatrix diagSdInv = new DiagonalMatrix(sdInv);
EigenDecomposition eigen = new EigenDecomposition(R);
RealMatrix eigenVectors = eigen.getV().getSubMatrix(0, nVariables - 1, 0, nFactors - 1);
double[] ev = new double[nFactors];
for (int i = 0; i < nFactors; i++) {
ev[i] = Math.sqrt(eigen.getRealEigenvalue(i));
}
DiagonalMatrix evMatrix =
new DiagonalMatrix(
ev); // USE Apache version of Diagonal matrix when upgrade to version 3.2
RealMatrix LAMBDA = eigenVectors.multiply(evMatrix);
RealMatrix SIGMA = (LAMBDA.multiply(LAMBDA.transpose()));
double value = 0.0;
RealMatrix DIF = R.subtract(SIGMA);
for (int i = 0; i < DIF.getRowDimension(); i++) {
for (int j = 0; j < DIF.getColumnDimension(); j++) {
value = DIF.getEntry(i, j);
DIF.setEntry(i, j, Math.pow(value, 2));
}
}
RealMatrix RESID = diagSdInv.multiply(DIF).multiply(diagSdInv);
double sum = 0.0;
for (int i = 0; i < RESID.getRowDimension(); i++) {
for (int j = 0; j < RESID.getColumnDimension(); j++) {
sum += RESID.getEntry(i, j);
}
}
return sum;
}
public double[][] squaredResiduals() {
double[][] resid = new double[nItems][nItems];
double temp = 0.0;
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
temp = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
resid[i][j] = temp * temp;
}
}
return resid;
}
public double sumSquaredElements(RealMatrix matrix) {
double sum = 0.0;
double v = 0.0;
for (int i = 0; i < matrix.getRowDimension(); i++) {
for (int j = 0; j < matrix.getColumnDimension(); j++) {
v = matrix.getEntry(i, j);
sum += (v * v);
}
}
return sum;
}
public double meanSquaredResidual() {
double ni = Double.valueOf(nItems).doubleValue();
double temp = 0.0, sum = 0.0;
for (int i = 0; i < SIGMA.getRowDimension(); i++) {
for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
temp = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
sum += temp * temp;
}
}
return sum / (ni * ni);
}
private static double[] calculateColumnInverseMeans(RealMatrix matrix) {
return IntStream.range(0, matrix.getColumnDimension())
.mapToDouble(
i ->
1.0
/ IntStream.range(0, matrix.getRowDimension())
.mapToDouble(j -> matrix.getEntry(j, i))
.average()
.orElseThrow(
() ->
new IllegalArgumentException(
"cannot calculate a average for column " + i)))
.toArray();
}
private static double[][] log2ColumnMeanCenteredCounts(
RealMatrix matrix, double[] columnInverseMeans) {
final double[][] result = new double[matrix.getRowDimension()][columnInverseMeans.length];
// First we divide by column mean then becoming a ratio;
// We also log_2 transform it at this point.
for (int i = 0; i < result.length; i++) {
final double[] inputRow = matrix.getRow(i);
for (int j = 0; j < columnInverseMeans.length; j++) {
if ((result[i][j] = inputRow[j] * columnInverseMeans[j]) < EPSILON) {
result[i][j] = EPSILON;
}
result[i][j] = Math.log(result[i][j]) * INV_LN2;
}
}
return result;
}
/**
* Create a PearsonsCorrelation from a RealMatrix whose columns represent variables to be
* correlated.
*
* <p>Throws MathIllegalArgumentException if the matrix does not have at least two columns and two
* rows. Pairwise correlations are set to NaN if one of the correlates has zero variance.
*
* @param matrix matrix with columns representing variables to correlate
* @throws MathIllegalArgumentException if the matrix does not contain sufficient data
* @see #correlation(double[], double[])
*/
public PearsonsCorrelation(RealMatrix matrix) {
nObs = matrix.getRowDimension();
correlationMatrix = computeCorrelationMatrix(matrix);
}
@Override
public List<MLCallbackResult> detect(
final String user,
final String algorithm,
UserActivityAggModel userActivity,
UserProfileEigenModel aModel) {
RealMatrix inputData = userActivity.matrix();
LOG.warn(
"EigenBasedAnomalyDetection predictAnomaly called with dimension: "
+ inputData.getRowDimension()
+ "x"
+ inputData.getColumnDimension());
if (aModel == null) {
LOG.warn(
"nothing to do as the input model does not have required values, returning from evaluating this algorithm..");
return null;
}
List<MLCallbackResult> mlCallbackResults = new ArrayList<MLCallbackResult>();
RealMatrix normalizedMat = normalizeData(inputData, aModel);
UserCommandStatistics[] listStats = aModel.statistics();
int colWithHighVariant = 0;
for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
if (listStats[j].isLowVariant() == false) {
colWithHighVariant++;
}
}
final Map<String, String> context =
new HashMap<String, String>() {
{
put(UserProfileConstants.USER_TAG, user);
put(UserProfileConstants.ALGORITHM_TAG, algorithm);
}
};
Map<Integer, String> lineNoWithVariantBasedAnomalyDetection = new HashMap<Integer, String>();
for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
MLCallbackResult aResult = new MLCallbackResult();
aResult.setContext(context);
for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
// LOG.info("mean for j=" + j + " is:" + listStats[j].getMean());
// LOG.info("stddev for j=" + j + " is:" + listStats[j].getStddev());
if (listStats[j].isLowVariant() == true) {
// LOG.info(listOfCmds[j] + " is low variant");
if (normalizedMat.getEntry(i, j) > listStats[j].getMean()) {
lineNoWithVariantBasedAnomalyDetection.put(i, "lowVariantAnomaly");
aResult.setAnomaly(true);
aResult.setTimestamp(userActivity.timestamp());
aResult.setFeature(listStats[j].getCommandName());
aResult.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
List<String> datapoints = new ArrayList<String>();
double[] rowVals = inputData.getRow(i);
for (double rowVal : rowVals) datapoints.add(rowVal + "");
aResult.setDatapoints(datapoints);
aResult.setId(user);
mlCallbackResults.add(aResult);
} else {
aResult.setAnomaly(false);
aResult.setTimestamp(userActivity.timestamp());
mlCallbackResults.add(aResult);
}
}
}
// return results;
}
// LOG.info("results size here: " + results.length);
// LOG.info("col with high variant: " + colWithHighVariant);
RealMatrix finalMatWithoutLowVariantFeatures =
new Array2DRowRealMatrix(normalizedMat.getRowDimension(), colWithHighVariant);
// LOG.info("size of final test data: " + finalMatWithoutLowVariantFeatures.getRowDimension()
// +"x"+ finalMatWithoutLowVariantFeatures.getColumnDimension());
int finalMatrixRow = 0;
int finalMatrixCol = 0;
for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
if (listStats[j].isLowVariant() == false) {
finalMatWithoutLowVariantFeatures.setEntry(
finalMatrixRow, finalMatrixCol, normalizedMat.getEntry(i, j));
finalMatrixCol++;
}
}
finalMatrixCol = 0;
finalMatrixRow++;
}
RealVector[] pcs = aModel.principalComponents();
// LOG.info("pc size: " + pcs.getRowDimension() +"x" + pcs.getColumnDimension());
RealMatrix finalInputMatTranspose = finalMatWithoutLowVariantFeatures.transpose();
for (int i = 0; i < finalMatWithoutLowVariantFeatures.getRowDimension(); i++) {
if (lineNoWithVariantBasedAnomalyDetection.get(i) == null) {
MLCallbackResult result = new MLCallbackResult();
result.setContext(context);
for (int sz = 0; sz < pcs.length; sz++) {
double[] pc1 = pcs[sz].toArray();
RealMatrix pc1Mat = new Array2DRowRealMatrix(pc1);
RealMatrix transposePC1Mat = pc1Mat.transpose();
RealMatrix testData =
pc1Mat.multiply(transposePC1Mat).multiply(finalInputMatTranspose.getColumnMatrix(i));
// LOG.info("testData size: " + testData.getRowDimension() + "x" +
// testData.getColumnDimension());
RealMatrix testDataTranspose = testData.transpose();
// LOG.info("testData transpose size: " + testDataTranspose.getRowDimension() + "x" +
// testDataTranspose.getColumnDimension());
RealVector iRowVector = testDataTranspose.getRowVector(0);
// RealVector pc1Vector = transposePC1Mat.getRowVector(sz);
RealVector pc1Vector = transposePC1Mat.getRowVector(0);
double distanceiRowAndPC1 = iRowVector.getDistance(pc1Vector);
// LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " +
// model.getMaxL2Norm().getEntry(sz));
// LOG.info("model.getMaxL2Norm().getEntry(sz):" + model.getMaxL2Norm().getEntry(sz));
if (distanceiRowAndPC1 > aModel.maximumL2Norm().getEntry(sz)) {
// LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " +
// model.getMaxL2Norm().getEntry(sz));
result.setAnomaly(true);
result.setFeature(aModel.statistics()[sz].getCommandName());
result.setTimestamp(System.currentTimeMillis());
result.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
List<String> datapoints = new ArrayList<String>();
double[] rowVals = inputData.getRow(i);
for (double rowVal : rowVals) datapoints.add(rowVal + "");
result.setDatapoints(datapoints);
result.setId(user);
}
}
mlCallbackResults.add(result);
}
}
return mlCallbackResults;
}
/**
* Tangent normalize a coverage profile.
*
* <p>Notes about the Spark tangent normalization can be found in docs/PoN/
*
* @param pon Not {@code null}
* @param targetFactorNormalizedCounts ReadCountCollection of counts that have already been
* normalized fully (typically, including the target factor normalization). I.e. a coverage
* profile The column names should be intact. Not {@code null} See {@link
* TangentNormalizer::createCoverageProfile}
* @return never {@code null}
*/
private static TangentNormalizationResult tangentNormalize(
final PoN pon, final ReadCountCollection targetFactorNormalizedCounts, JavaSparkContext ctx) {
Utils.nonNull(pon, "PoN cannot be null.");
Utils.nonNull(targetFactorNormalizedCounts, "targetFactorNormalizedCounts cannot be null.");
Utils.nonNull(
targetFactorNormalizedCounts.columnNames(),
"targetFactorNormalizedCounts column names cannot be null.");
ParamUtils.isPositive(
targetFactorNormalizedCounts.columnNames().size(),
"targetFactorNormalizedCounts column names cannot be an empty list.");
final Case2PoNTargetMapper targetMapper =
new Case2PoNTargetMapper(targetFactorNormalizedCounts.targets(), pon.getPanelTargetNames());
// The input counts with rows (targets) sorted so that they match the PoN's order.
final RealMatrix tangentNormalizationRawInputCounts =
targetMapper.fromCaseToPoNCounts(targetFactorNormalizedCounts.counts());
// We prepare the counts for tangent normalization.
final RealMatrix tangentNormalizationInputCounts =
composeTangentNormalizationInputMatrix(tangentNormalizationRawInputCounts);
if (ctx == null) {
// Calculate the beta-hats for the input read count columns (samples).
logger.info("Calculating beta hats...");
final RealMatrix tangentBetaHats =
pon.betaHats(tangentNormalizationInputCounts, true, EPSILON);
// Actual tangent normalization step.
logger.info(
"Performing actual tangent normalization ("
+ tangentNormalizationInputCounts.getColumnDimension()
+ " columns)...");
final RealMatrix tangentNormalizedCounts =
pon.tangentNormalization(tangentNormalizationInputCounts, tangentBetaHats, true);
// Output the tangent normalized counts.
logger.info("Post-processing tangent normalization results...");
final ReadCountCollection tangentNormalized =
targetMapper.fromPoNtoCaseCountCollection(
tangentNormalizedCounts, targetFactorNormalizedCounts.columnNames());
final ReadCountCollection preTangentNormalized =
targetMapper.fromPoNtoCaseCountCollection(
tangentNormalizationInputCounts, targetFactorNormalizedCounts.columnNames());
return new TangentNormalizationResult(
tangentNormalized, preTangentNormalized, tangentBetaHats, targetFactorNormalizedCounts);
} else {
/*
Using Spark: the code here is a little more complex for optimization purposes.
Please see notes in docs/PoN ...
Ahat^T = (C^T P^T) A^T
Therefore, C^T is the RowMatrix
pinv: P
panel: A
projection: Ahat
cases: C
betahat: C^T P^T
tangentNormalizedCounts: C - Ahat
*/
final RealMatrix pinv = pon.getReducedPanelPInverseCounts();
final RealMatrix panel = pon.getReducedPanelCounts();
// Make the C^T a distributed matrix (RowMatrix)
final RowMatrix caseTDistMat =
SparkConverter.convertRealMatrixToSparkRowMatrix(
ctx, tangentNormalizationInputCounts.transpose(), TN_NUM_SLICES_SPARK);
// Spark local matrices (transposed)
final Matrix pinvTLocalMat =
new DenseMatrix(
pinv.getRowDimension(),
pinv.getColumnDimension(),
Doubles.concat(pinv.getData()),
true)
.transpose();
final Matrix panelTLocalMat =
new DenseMatrix(
panel.getRowDimension(),
panel.getColumnDimension(),
Doubles.concat(panel.getData()),
true)
.transpose();
// Calculate the projection transpose in a distributed matrix, then convert to Apache Commons
// matrix (not transposed)
final RowMatrix betahatDistMat = caseTDistMat.multiply(pinvTLocalMat);
final RowMatrix projectionTDistMat = betahatDistMat.multiply(panelTLocalMat);
final RealMatrix projection =
SparkConverter.convertSparkRowMatrixToRealMatrix(
projectionTDistMat, tangentNormalizationInputCounts.transpose().getRowDimension())
.transpose();
// Subtract the cases from the projection
final RealMatrix tangentNormalizedCounts =
tangentNormalizationInputCounts.subtract(projection);
// Construct the result object and return it with the correct targets.
final ReadCountCollection tangentNormalized =
targetMapper.fromPoNtoCaseCountCollection(
tangentNormalizedCounts, targetFactorNormalizedCounts.columnNames());
final ReadCountCollection preTangentNormalized =
targetMapper.fromPoNtoCaseCountCollection(
tangentNormalizationInputCounts, targetFactorNormalizedCounts.columnNames());
final RealMatrix tangentBetaHats =
SparkConverter.convertSparkRowMatrixToRealMatrix(
betahatDistMat, tangentNormalizedCounts.getColumnDimension());
return new TangentNormalizationResult(
tangentNormalized,
preTangentNormalized,
tangentBetaHats.transpose(),
targetFactorNormalizedCounts);
}
}
/**
* Create a covariance matrix from a matrix whose columns represent covariates.
*
* <p>The <code>biasCorrected</code> parameter determines whether or not covariance estimates are
* bias-corrected.
*
* <p>The matrix must have at least one column and two rows
*
* @param matrix matrix with columns representing covariates
* @param biasCorrected true means covariances are bias-corrected
* @throws MathIllegalArgumentException if the input matrix does not have at least two rows and
* one column
*/
public Covariance(RealMatrix matrix, boolean biasCorrected) throws MathIllegalArgumentException {
checkSufficientData(matrix);
n = matrix.getRowDimension();
covarianceMatrix = computeCovarianceMatrix(matrix, biasCorrected);
}
@Override
public void process() {
// Get image dimensions, store ICA dimensions (number of frames).
int[] dim = ip.getDimensions();
dimensions = dim[4];
IJ.showStatus("ICA: reformatting data matrix...");
// Compute number of real voxels to be used
int n = 0;
for (@SuppressWarnings("unused") Voxel v : ip) {
n++;
}
// Create new array and fill it with the image data
double[][] image_data = new double[n][dimensions];
int i = 0;
for (Voxel v : ip) {
image_data[i++] = v.tac;
}
// Transpose the data matrix
Array2DRowRealMatrix temp = new Array2DRowRealMatrix(image_data, false);
image_data = temp.transpose().getData();
temp = null;
System.gc();
// Perform the ICA computation
IJ.showStatus("ICA: performing source separation " + "(may take some time)...");
FastICA fi = null;
try {
fi = new FastICA(image_data, ican);
} catch (FastICAException e) {
System.out.println(e.getLocalizedMessage());
}
IJ.showStatus("ICA computed, reformatting results...");
// Get projections on each dimension
double[][] vectors = fi.getICVectors();
// Get independent signals (to be offered as additional information)
// It is transposed to print the columns.
double[][] sep = fi.getSeparatingMatrix();
RealMatrix sources = new Array2DRowRealMatrix(sep);
sources = sources.transpose();
Array2DRowRealMatrix result = new Array2DRowRealMatrix(vectors, false);
// If the ICA image is to be shown, create a new image with
// result.getRowDimension() frames and the original number of
// x, y, z dimensions
if (showICA) {
ImagePlus ICA_image = RealMatrix2IJ(result, dim, this.ip, "ICA image");
ICA_image.show();
}
// Please note: this is somehow incorrect. As the clustering model
// that we are following needs one voxel -> one cluster, this step
// below assigns each voxel to the principal component with the
// maximum weight for its particular kinetics. In "real life", the
// resulting images would contain the contribution of that component
// in all voxels, but for segmentation purposes this approach is
// chosen.
int column_index = 0;
for (Voxel v : ip) {
double[] projection = result.getColumn(column_index++);
// Every Voxel belongs to the maximum index of its projected TAC
int max = getMaxIndex(projection) + 1;
addTACtoCluster(v, max);
}
// Fill in the additionalInfo array.
additionalInfo = new String[2];
additionalInfo[0] = "ica_sources";
StringBuilder sb = new StringBuilder();
int rows = sources.getRowDimension();
for (int j = 0; j < rows; j++) {
double[] row = sources.getRow(j);
sb.append(Arrays.toString(row));
sb.append("\n");
}
// Remove brackets
String tempstr = sb.toString();
tempstr = tempstr.replace("[", "");
tempstr = tempstr.replace("]", "");
additionalInfo[1] = tempstr;
}
/////////////////////////////////// Constructor ////////////////////////////////////////////////
public CostFunction(
List<Pixel> input,
List<Pixel> average,
List<Pixel> model,
RealMatrix inputFeatPts,
RealMatrix averageFeatPts,
RealMatrix modelFeatPts,
float[][] eigenVectorsS,
float[][] eigenVectorsT,
Bitmap bmpModel,
float[] inputAlpha,
float[] inputBeta,
float sigmaI,
float sigmaF) {
// this.featPtsIndex = readBin83PtIndex(CONFIG_DIRECTORY, INDEX83PT_FILE);
this.input = input;
this.average = average;
this.model = model;
this.inputFeatPts = inputFeatPts;
this.averageFeatPts = averageFeatPts;
this.modelFeatPts = modelFeatPts;
this.k = inputFeatPts.getRowDimension(); // should be equal to 83
this.num_points = input.size(); // should be equal to 8489
this.eigValS = readBinFloat(SHAPE_DIRECTORY, EIG_SHAPE_FILE, 60);
this.eigValT = readBinFloat(TEXTURE_DIRECTORY, EIG_TEXTURE_FILE, 100);
/*this.subFSV = readBinSFSV(CONFIG_DIRECTORY, SFSV_FILE);*/
this.landmarks83Index =
readBin83PtIndex(
CONFIG_DIRECTORY, INDEX83PT_FILE); // we access directly in featureShape file
// instead of using subFSV file
// Checking arguments
if (num_points != 8489) {
throw new IllegalArgumentException("num_points not equal 8489");
}
if (k != 83) {
throw new IllegalArgumentException("k not equal 83");
}
if (input.isEmpty() || average.isEmpty() || model.isEmpty()) {
throw new IllegalArgumentException("input or average or model list are empty");
}
if (input.size() != model.size()) {
throw new IllegalArgumentException("input and model list do not have the same size");
}
if (input.size() != average.size()) {
throw new IllegalArgumentException("input and average list do not have the same size");
}
if (average.size() != model.size()) {
throw new IllegalArgumentException("average and model list do not have the same size");
}
if (averageFeatPts.getRowDimension() != k
|| averageFeatPts.getColumnDimension() != inputFeatPts.getColumnDimension()) {
throw new IllegalArgumentException(
"inputFeatPts and averageFeatPts do not have the same size");
}
if (modelFeatPts.getRowDimension() != k
|| modelFeatPts.getColumnDimension() != inputFeatPts.getColumnDimension()) {
throw new IllegalArgumentException("inputFeatPts and modelFeatPts do not have the same size");
}
// Initialy populate list with 0, the value doesn't matter
for (int h = 0; h < 500; h++) {
this.randomList.add(0);
}
this.s = eigenVectorsS;
this.t = eigenVectorsT;
this.inputAlpha = inputAlpha;
this.inputBeta = inputBeta;
this.sigmaI = sigmaI;
this.sigmaF = sigmaF;
this.Iinput = computeIinput(); // is always the same
this.Imodel = computeImodel();
this.IinputBeta = computeIinputBeta(); // is always the same
this.ImodelBeta = computeImodelBeta();
this.dxModel = computeSobelGx(bmpModel);
this.dyModel = computeSobelGy(bmpModel);
// saveBitmaptoPNG(TEXTURE_DIRECTORY, "modelFace2DGx.png", bmpModelGx); //save
// saveBitmaptoPNG(TEXTURE_DIRECTORY, "modelFace2DGy.png", bmpModelGy); //save
this.E = computeE();
this.alpha = inputAlpha; // Initialize
this.beta = inputBeta; // Initialize
computeAlpha(); // Compute 60 alpha values output
computeBeta(); // Compute 100 beta values output
/* Collections.sort(randomList);
for(int idx : randomList) {
Log.d(TAG,"randomList = " + idx);
}*/
}