/**
* Function to perform LU decomposition on a given matrix.
*
* @param in matrix object
* @return array of matrix blocks
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock[] computeLU(MatrixObject in) throws DMLRuntimeException {
if (in.getNumRows() != in.getNumColumns()) {
throw new DMLRuntimeException(
"LU Decomposition can only be done on a square matrix. Input matrix is rectangular (rows="
+ in.getNumRows()
+ ", cols="
+ in.getNumColumns()
+ ")");
}
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);
// Perform LUP decomposition
LUDecomposition ludecompose = new LUDecomposition(matrixInput);
RealMatrix P = ludecompose.getP();
RealMatrix L = ludecompose.getL();
RealMatrix U = ludecompose.getU();
// Read the results into native format
MatrixBlock mbP = DataConverter.convertToMatrixBlock(P.getData());
MatrixBlock mbL = DataConverter.convertToMatrixBlock(L.getData());
MatrixBlock mbU = DataConverter.convertToMatrixBlock(U.getData());
return new MatrixBlock[] {mbP, mbL, mbU};
}
/**
* Function to perform QR decomposition on a given matrix.
*
* @param in matrix object
* @return array of matrix blocks
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock[] computeQR(MatrixObject in) throws DMLRuntimeException {
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);
// Perform QR decomposition
QRDecomposition qrdecompose = new QRDecomposition(matrixInput);
RealMatrix H = qrdecompose.getH();
RealMatrix R = qrdecompose.getR();
// Read the results into native format
MatrixBlock mbH = DataConverter.convertToMatrixBlock(H.getData());
MatrixBlock mbR = DataConverter.convertToMatrixBlock(R.getData());
return new MatrixBlock[] {mbH, mbR};
}
// turns 3X1 matrix back into coordinate instance (uses t)
private static Coordinate MatrixToCoordinateInstance(String system, RealMatrix V, double t) {
double[][] vec = V.getData();
double vecX = vec[0][0];
double vecY = vec[0][1];
double vecZ = vec[0][2];
return new Coordinate(system, vecX, vecY, vecZ, t);
}
public RealMatrix makeZta(RealMatrix userFeature, RealMatrix articleFeature) {
RealMatrix product = userFeature.multiply(articleFeature.transpose());
double[][] productData = product.getData();
double[] productVector = new double[36];
int count = 0;
for (int row = 0; row < 6; row++) {
for (int col = 0; col < 6; col++) {
productVector[count] = productData[row][col];
count++;
}
}
return MatrixUtils.createColumnRealMatrix(productVector);
}
/**
* Function to compute Cholesky decomposition of the given input matrix. The input must be a real
* symmetric positive-definite matrix.
*
* @param in commons-math3 Array2DRowRealMatrix
* @return matrix block
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock computeCholesky(Array2DRowRealMatrix in) throws DMLRuntimeException {
if (!in.isSquare())
throw new DMLRuntimeException(
"Input to cholesky() must be square matrix -- given: a "
+ in.getRowDimension()
+ "x"
+ in.getColumnDimension()
+ " matrix.");
CholeskyDecomposition cholesky = new CholeskyDecomposition(in);
RealMatrix rmL = cholesky.getL();
return DataConverter.convertToMatrixBlock(rmL.getData());
}
/**
* Function to compute matrix inverse via matrix decomposition.
*
* @param in commons-math3 Array2DRowRealMatrix
* @return matrix block
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock computeMatrixInverse(Array2DRowRealMatrix in)
throws DMLRuntimeException {
if (!in.isSquare())
throw new DMLRuntimeException(
"Input to inv() must be square matrix -- given: a "
+ in.getRowDimension()
+ "x"
+ in.getColumnDimension()
+ " matrix.");
QRDecomposition qrdecompose = new QRDecomposition(in);
DecompositionSolver solver = qrdecompose.getSolver();
RealMatrix inverseMatrix = solver.getInverse();
return DataConverter.convertToMatrixBlock(inverseMatrix.getData());
}
/**
* Function to solve a given system of equations.
*
* @param in1 matrix object 1
* @param in2 matrix object 2
* @return matrix block
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock computeSolve(MatrixObject in1, MatrixObject in2)
throws DMLRuntimeException {
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in1);
Array2DRowRealMatrix vectorInput = DataConverter.convertToArray2DRowRealMatrix(in2);
/*LUDecompositionImpl ludecompose = new LUDecompositionImpl(matrixInput);
DecompositionSolver lusolver = ludecompose.getSolver();
RealMatrix solutionMatrix = lusolver.solve(vectorInput);*/
// Setup a solver based on QR Decomposition
QRDecomposition qrdecompose = new QRDecomposition(matrixInput);
DecompositionSolver solver = qrdecompose.getSolver();
// Invoke solve
RealMatrix solutionMatrix = solver.solve(vectorInput);
return DataConverter.convertToMatrixBlock(solutionMatrix.getData());
}
/**
* Function to perform Eigen decomposition on a given matrix. Input must be a symmetric matrix.
*
* @param in matrix object
* @return array of matrix blocks
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock[] computeEigen(MatrixObject in) throws DMLRuntimeException {
if (in.getNumRows() != in.getNumColumns()) {
throw new DMLRuntimeException(
"Eigen Decomposition can only be done on a square matrix. Input matrix is rectangular (rows="
+ in.getNumRows()
+ ", cols="
+ in.getNumColumns()
+ ")");
}
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);
EigenDecomposition eigendecompose = new EigenDecomposition(matrixInput);
RealMatrix eVectorsMatrix = eigendecompose.getV();
double[][] eVectors = eVectorsMatrix.getData();
double[] eValues = eigendecompose.getRealEigenvalues();
// Sort the eigen values (and vectors) in increasing order (to be compatible w/ LAPACK.DSYEVR())
int n = eValues.length;
for (int i = 0; i < n; i++) {
int k = i;
double p = eValues[i];
for (int j = i + 1; j < n; j++) {
if (eValues[j] < p) {
k = j;
p = eValues[j];
}
}
if (k != i) {
eValues[k] = eValues[i];
eValues[i] = p;
for (int j = 0; j < n; j++) {
p = eVectors[j][i];
eVectors[j][i] = eVectors[j][k];
eVectors[j][k] = p;
}
}
}
MatrixBlock mbValues = DataConverter.convertToMatrixBlock(eValues, true);
MatrixBlock mbVectors = DataConverter.convertToMatrixBlock(eVectors);
return new MatrixBlock[] {mbValues, mbVectors};
}
@Override
public double[][] correlate(
List<DataSet> dataSets, Timestamp startDate, Timestamp endDate, TimeUnits granularity) {
if (dataSets == null || dataSets.size() < 2) {
throw new IllegalArgumentException("At least 2 data sets are required for correlation");
}
final Map<Long, List<Double>> orderedPoints = new TreeMap<Long, List<Double>>();
for (int i = 0; i < dataSets.size(); i++) {
final DataSet dataSet = dataSets.get(i);
final List<DataSetPoint> points =
this.dao.findAndAggregatePointsGroupedByUnit(dataSet, startDate, endDate, granularity);
for (final DataSetPoint point : points) {
final Long millis = point.getX().getTime();
if (!orderedPoints.containsKey(millis)) {
orderedPoints.put(millis, newDoubleList(dataSets.size()));
}
orderedPoints.get(millis).set(i, point.getY());
}
}
if (orderedPoints.size() <= 1) {
throw new IllegalArgumentException(
"Needed at least 2 points, found: " + orderedPoints.size());
}
final double[][] points = new double[orderedPoints.size()][dataSets.size()];
int i = 0;
for (final Map.Entry<Long, List<Double>> entry : orderedPoints.entrySet()) {
final List<Double> values = entry.getValue();
for (int j = 0; j < values.size(); j++) {
points[i][j] = values.get(j);
}
i++;
}
final PearsonsCorrelation correlation = new PearsonsCorrelation();
final RealMatrix matrix = correlation.computeCorrelationMatrix(points);
return matrix.getData();
}
/* 91: */
/* 92: */ public Array2DRowRealMatrix initializeHighOrderDerivatives(
double h, double[] t, double[][] y, double[][] yDot)
/* 93: */ {
/* 94:259 */ double[][] a = new double[2 * (y.length - 1)][this.c1.length];
/* 95:260 */ double[][] b = new double[2 * (y.length - 1)][y[0].length];
/* 96:261 */ double[] y0 = y[0];
/* 97:262 */ double[] yDot0 = yDot[0];
/* 98:263 */ for (int i = 1; i < y.length; i++)
/* 99: */ {
/* 100:265 */ double di = t[i] - t[0];
/* 101:266 */ double ratio = di / h;
/* 102:267 */ double dikM1Ohk = 1.0D / h;
/* 103: */
/* 104: */
/* 105: */
/* 106:271 */ double[] aI = a[(2 * i - 2)];
/* 107:272 */ double[] aDotI = a[(2 * i - 1)];
/* 108:273 */ for (int j = 0; j < aI.length; j++)
/* 109: */ {
/* 110:274 */ dikM1Ohk *= ratio;
/* 111:275 */ aI[j] = (di * dikM1Ohk);
/* 112:276 */ aDotI[j] = ((j + 2) * dikM1Ohk);
/* 113: */ }
/* 114:280 */ double[] yI = y[i];
/* 115:281 */ double[] yDotI = yDot[i];
/* 116:282 */ double[] bI = b[(2 * i - 2)];
/* 117:283 */ double[] bDotI = b[(2 * i - 1)];
/* 118:284 */ for (int j = 0; j < yI.length; j++)
/* 119: */ {
/* 120:285 */ bI[j] = (yI[j] - y0[j] - di * yDot0[j]);
/* 121:286 */ yDotI[j] -= yDot0[j];
/* 122: */ }
/* 123: */ }
/* 124:294 */ QRDecomposition decomposition =
new QRDecomposition(new Array2DRowRealMatrix(a, false));
/* 125:295 */ RealMatrix x =
decomposition.getSolver().solve(new Array2DRowRealMatrix(b, false));
/* 126:296 */ return new Array2DRowRealMatrix(x.getData(), false);
/* 127: */ }
@Override
public Article chooseArm(User user, List<Article> articles) {
Article bestA = null;
double bestArmP = Double.MIN_VALUE;
RealMatrix Aa;
RealMatrix Ba;
RealMatrix ba;
for (Article a : articles) {
String aId = a.getId();
if (!AMap.containsKey(aId)) {
Aa = MatrixUtils.createRealIdentityMatrix(6);
AMap.put(aId, Aa); // set as identity for now and we will update
// in reward
double[] zeros = {0, 0, 0, 0, 0, 0};
ba = MatrixUtils.createColumnRealMatrix(zeros);
bMap.put(aId, ba);
double[][] BMapZeros = new double[6][36];
for (double[] row : BMapZeros) {
Arrays.fill(row, 0.0);
}
Ba = MatrixUtils.createRealMatrix(BMapZeros);
BMap.put(aId, Ba);
} else {
Aa = AMap.get(aId);
ba = bMap.get(aId);
Ba = BMap.get(aId);
}
// Make column vector out of features
RealMatrix xta = MatrixUtils.createColumnRealMatrix(a.getFeatures());
RealMatrix zta = makeZta(MatrixUtils.createColumnRealMatrix(user.getFeatures()), xta);
// Set up common variables
RealMatrix A0Inverse = MatrixUtils.inverse(A0);
RealMatrix AaInverse = MatrixUtils.inverse(Aa);
RealMatrix ztaTranspose = zta.transpose();
RealMatrix BaTranspose = Ba.transpose();
RealMatrix xtaTranspose = xta.transpose();
// Find theta
RealMatrix theta = AaInverse.multiply(ba.subtract(Ba.multiply(BetaHat)));
// Find sta
RealMatrix staMatrix = ztaTranspose.multiply(A0Inverse).multiply(zta);
staMatrix =
staMatrix.subtract(
ztaTranspose
.multiply(A0Inverse)
.multiply(BaTranspose)
.multiply(AaInverse)
.multiply(xta)
.scalarMultiply(2));
staMatrix = staMatrix.add(xtaTranspose.multiply(AaInverse).multiply(xta));
staMatrix =
staMatrix.add(
xtaTranspose
.multiply(AaInverse)
.multiply(Ba)
.multiply(A0Inverse)
.multiply(BaTranspose)
.multiply(AaInverse)
.multiply(xta));
// Find pta for arm
RealMatrix ptaMatrix = ztaTranspose.multiply(BetaHat);
ptaMatrix = ptaMatrix.add(xtaTranspose.multiply(theta));
double ptaVal = ptaMatrix.getData()[0][0];
double staVal = staMatrix.getData()[0][0];
ptaVal = ptaVal + alpha * Math.sqrt(staVal);
// Update argmax
if (ptaVal > bestArmP) {
bestArmP = ptaVal;
bestA = a;
}
}
return bestA;
}
/**
* Tangent normalize a coverage profile.
*
* <p>Notes about the Spark tangent normalization can be found in docs/PoN/
*
* @param pon Not {@code null}
* @param targetFactorNormalizedCounts ReadCountCollection of counts that have already been
* normalized fully (typically, including the target factor normalization). I.e. a coverage
* profile The column names should be intact. Not {@code null} See {@link
* TangentNormalizer::createCoverageProfile}
* @return never {@code null}
*/
private static TangentNormalizationResult tangentNormalize(
final PoN pon, final ReadCountCollection targetFactorNormalizedCounts, JavaSparkContext ctx) {
Utils.nonNull(pon, "PoN cannot be null.");
Utils.nonNull(targetFactorNormalizedCounts, "targetFactorNormalizedCounts cannot be null.");
Utils.nonNull(
targetFactorNormalizedCounts.columnNames(),
"targetFactorNormalizedCounts column names cannot be null.");
ParamUtils.isPositive(
targetFactorNormalizedCounts.columnNames().size(),
"targetFactorNormalizedCounts column names cannot be an empty list.");
final Case2PoNTargetMapper targetMapper =
new Case2PoNTargetMapper(targetFactorNormalizedCounts.targets(), pon.getPanelTargetNames());
// The input counts with rows (targets) sorted so that they match the PoN's order.
final RealMatrix tangentNormalizationRawInputCounts =
targetMapper.fromCaseToPoNCounts(targetFactorNormalizedCounts.counts());
// We prepare the counts for tangent normalization.
final RealMatrix tangentNormalizationInputCounts =
composeTangentNormalizationInputMatrix(tangentNormalizationRawInputCounts);
if (ctx == null) {
// Calculate the beta-hats for the input read count columns (samples).
logger.info("Calculating beta hats...");
final RealMatrix tangentBetaHats =
pon.betaHats(tangentNormalizationInputCounts, true, EPSILON);
// Actual tangent normalization step.
logger.info(
"Performing actual tangent normalization ("
+ tangentNormalizationInputCounts.getColumnDimension()
+ " columns)...");
final RealMatrix tangentNormalizedCounts =
pon.tangentNormalization(tangentNormalizationInputCounts, tangentBetaHats, true);
// Output the tangent normalized counts.
logger.info("Post-processing tangent normalization results...");
final ReadCountCollection tangentNormalized =
targetMapper.fromPoNtoCaseCountCollection(
tangentNormalizedCounts, targetFactorNormalizedCounts.columnNames());
final ReadCountCollection preTangentNormalized =
targetMapper.fromPoNtoCaseCountCollection(
tangentNormalizationInputCounts, targetFactorNormalizedCounts.columnNames());
return new TangentNormalizationResult(
tangentNormalized, preTangentNormalized, tangentBetaHats, targetFactorNormalizedCounts);
} else {
/*
Using Spark: the code here is a little more complex for optimization purposes.
Please see notes in docs/PoN ...
Ahat^T = (C^T P^T) A^T
Therefore, C^T is the RowMatrix
pinv: P
panel: A
projection: Ahat
cases: C
betahat: C^T P^T
tangentNormalizedCounts: C - Ahat
*/
final RealMatrix pinv = pon.getReducedPanelPInverseCounts();
final RealMatrix panel = pon.getReducedPanelCounts();
// Make the C^T a distributed matrix (RowMatrix)
final RowMatrix caseTDistMat =
SparkConverter.convertRealMatrixToSparkRowMatrix(
ctx, tangentNormalizationInputCounts.transpose(), TN_NUM_SLICES_SPARK);
// Spark local matrices (transposed)
final Matrix pinvTLocalMat =
new DenseMatrix(
pinv.getRowDimension(),
pinv.getColumnDimension(),
Doubles.concat(pinv.getData()),
true)
.transpose();
final Matrix panelTLocalMat =
new DenseMatrix(
panel.getRowDimension(),
panel.getColumnDimension(),
Doubles.concat(panel.getData()),
true)
.transpose();
// Calculate the projection transpose in a distributed matrix, then convert to Apache Commons
// matrix (not transposed)
final RowMatrix betahatDistMat = caseTDistMat.multiply(pinvTLocalMat);
final RowMatrix projectionTDistMat = betahatDistMat.multiply(panelTLocalMat);
final RealMatrix projection =
SparkConverter.convertSparkRowMatrixToRealMatrix(
projectionTDistMat, tangentNormalizationInputCounts.transpose().getRowDimension())
.transpose();
// Subtract the cases from the projection
final RealMatrix tangentNormalizedCounts =
tangentNormalizationInputCounts.subtract(projection);
// Construct the result object and return it with the correct targets.
final ReadCountCollection tangentNormalized =
targetMapper.fromPoNtoCaseCountCollection(
tangentNormalizedCounts, targetFactorNormalizedCounts.columnNames());
final ReadCountCollection preTangentNormalized =
targetMapper.fromPoNtoCaseCountCollection(
tangentNormalizationInputCounts, targetFactorNormalizedCounts.columnNames());
final RealMatrix tangentBetaHats =
SparkConverter.convertSparkRowMatrixToRealMatrix(
betahatDistMat, tangentNormalizedCounts.getColumnDimension());
return new TangentNormalizationResult(
tangentNormalized,
preTangentNormalized,
tangentBetaHats.transpose(),
targetFactorNormalizedCounts);
}
}
private double generalizedCorrelationRatio(SampleIterator it, int inputDim, int out) {
Map<Double, Integer> n_y = new HashMap<>();
Map<Double, MultivariateSummaryStatistics> stat_y = new HashMap<>();
List<RealMatrix> x = new ArrayList<>();
MultivariateSummaryStatistics stat = new MultivariateSummaryStatistics(inputDim, unbiased);
for (int i = 0; i < maxSamples && it.hasNext(); i++) {
Sample sample = it.next();
double[] input = sample.getEncodedInput().toArray();
double output = sample.getEncodedOutput().getEntry(out);
if (!n_y.containsKey(output)) {
n_y.put(output, 0);
stat_y.put(output, new MultivariateSummaryStatistics(inputDim, unbiased));
}
injectNoise(input);
n_y.put(output, n_y.get(output) + 1);
stat_y.get(output).addValue(input);
x.add(new Array2DRowRealMatrix(input));
stat.addValue(input);
}
RealMatrix x_sum = new Array2DRowRealMatrix(stat.getSum());
Map<Double, RealMatrix> x_y_sum = new HashMap<>();
for (Entry<Double, MultivariateSummaryStatistics> entry : stat_y.entrySet()) {
x_y_sum.put(entry.getKey(), new Array2DRowRealMatrix(entry.getValue().getSum()));
}
RealMatrix H = new Array2DRowRealMatrix(inputDim, inputDim);
RealMatrix temp = new Array2DRowRealMatrix(inputDim, inputDim);
for (double key : n_y.keySet()) {
temp =
temp.add(
x_y_sum
.get(key)
.multiply(x_y_sum.get(key).transpose())
.scalarMultiply(1.0 / n_y.get(key)));
}
H = temp.subtract(x_sum.multiply(x_sum.transpose()).scalarMultiply(1.0 / x.size()));
RealMatrix E = new Array2DRowRealMatrix(inputDim, inputDim);
for (RealMatrix m : x) {
E = E.add(m.multiply(m.transpose()));
}
E = E.subtract(temp);
List<Integer> zeroColumns = findZeroColumns(E);
E = removeZeroColumns(E, zeroColumns);
H = removeZeroColumns(H, zeroColumns);
Matrix JE = new Matrix(E.getData());
Matrix JH = new Matrix(H.getData());
if (JE.rank() < JE.getRowDimension()) {
Log.write(this, "Some error occurred (E matrix is singular)");
return -1;
} else {
double lambda;
if (useEigenvalues) {
Matrix L = JE.inverse().times(JH);
double[] eigs = L.eig().getRealEigenvalues();
Arrays.sort(eigs);
lambda = 1;
int nonNullEigs = n_y.keySet().size() - 1;
for (int i = eigs.length - nonNullEigs; i < eigs.length; i++) {
if (Math.abs(eigs[i]) < zeroThreshold) {
Log.write(this, "Some error occurred (E matrix has too many null eigenvalues)");
return -1;
}
lambda *= 1.0 / (1.0 + eigs[i]);
}
} else {
Matrix sum = JE.plus(JH);
if (sum.rank() < sum.getRowDimension()) {
Log.write(this, "Some error occourred (E+H is singular");
return -1;
}
lambda = JE.det() / sum.det();
}
return Math.sqrt(1 - lambda);
}
}