/**
* Function to perform LU decomposition on a given matrix.
*
* @param in matrix object
* @return array of matrix blocks
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock[] computeLU(MatrixObject in) throws DMLRuntimeException {
if (in.getNumRows() != in.getNumColumns()) {
throw new DMLRuntimeException(
"LU Decomposition can only be done on a square matrix. Input matrix is rectangular (rows="
+ in.getNumRows()
+ ", cols="
+ in.getNumColumns()
+ ")");
}
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);
// Perform LUP decomposition
LUDecomposition ludecompose = new LUDecomposition(matrixInput);
RealMatrix P = ludecompose.getP();
RealMatrix L = ludecompose.getL();
RealMatrix U = ludecompose.getU();
// Read the results into native format
MatrixBlock mbP = DataConverter.convertToMatrixBlock(P.getData());
MatrixBlock mbL = DataConverter.convertToMatrixBlock(L.getData());
MatrixBlock mbU = DataConverter.convertToMatrixBlock(U.getData());
return new MatrixBlock[] {mbP, mbL, mbU};
}
/**
* @param vector
* @param singleColBlock
* @param dense
* @param unknownDims
*/
private void testDataFrameConversion(
ValueType[] schema, boolean containsID, boolean dense, boolean unknownDims) {
boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform;
SparkExecutionContext sec = null;
try {
DMLScript.USE_LOCAL_SPARK_CONFIG = true;
DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
// generate input data and setup metadata
int cols = schema.length + colsVector - 1;
double sparsity = dense ? sparsity1 : sparsity2;
double[][] A = TestUtils.round(getRandomMatrix(rows1, cols, -10, 1000, sparsity, 2373));
MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc1 =
new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros());
MatrixCharacteristics mc2 =
unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
// setup spark context
sec = (SparkExecutionContext) ExecutionContextFactory.createContext();
JavaSparkContext sc = sec.getSparkContext();
SQLContext sqlctx = new SQLContext(sc);
// create input data frame
DataFrame df = createDataFrame(sqlctx, mbA, containsID, schema);
// dataframe - frame conversion
JavaPairRDD<Long, FrameBlock> out =
FrameRDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc2, containsID);
// get output frame block
FrameBlock fbB =
SparkExecutionContext.toFrameBlock(
out, UtilFunctions.nCopies(cols, ValueType.DOUBLE), rows1, cols);
// compare frame blocks
MatrixBlock mbB = DataConverter.convertToMatrixBlock(fbB);
double[][] B = DataConverter.convertToDoubleMatrix(mbB);
TestUtils.compareMatrices(A, B, rows1, cols, eps);
} catch (Exception ex) {
throw new RuntimeException(ex);
} finally {
sec.close();
DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig;
DMLScript.rtplatform = oldPlatform;
}
}
/**
* Function to perform QR decomposition on a given matrix.
*
* @param in matrix object
* @return array of matrix blocks
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock[] computeQR(MatrixObject in) throws DMLRuntimeException {
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);
// Perform QR decomposition
QRDecomposition qrdecompose = new QRDecomposition(matrixInput);
RealMatrix H = qrdecompose.getH();
RealMatrix R = qrdecompose.getR();
// Read the results into native format
MatrixBlock mbH = DataConverter.convertToMatrixBlock(H.getData());
MatrixBlock mbR = DataConverter.convertToMatrixBlock(R.getData());
return new MatrixBlock[] {mbH, mbR};
}
public static MatrixBlock unaryOperations(MatrixObject inj, String opcode)
throws DMLRuntimeException {
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(inj);
if (opcode.equals("inverse")) return computeMatrixInverse(matrixInput);
else if (opcode.equals("cholesky")) return computeCholesky(matrixInput);
return null;
}
/**
* Function to solve a given system of equations.
*
* @param in1 matrix object 1
* @param in2 matrix object 2
* @return matrix block
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock computeSolve(MatrixObject in1, MatrixObject in2)
throws DMLRuntimeException {
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in1);
Array2DRowRealMatrix vectorInput = DataConverter.convertToArray2DRowRealMatrix(in2);
/*LUDecompositionImpl ludecompose = new LUDecompositionImpl(matrixInput);
DecompositionSolver lusolver = ludecompose.getSolver();
RealMatrix solutionMatrix = lusolver.solve(vectorInput);*/
// Setup a solver based on QR Decomposition
QRDecomposition qrdecompose = new QRDecomposition(matrixInput);
DecompositionSolver solver = qrdecompose.getSolver();
// Invoke solve
RealMatrix solutionMatrix = solver.solve(vectorInput);
return DataConverter.convertToMatrixBlock(solutionMatrix.getData());
}
/**
* @param sqlctx
* @param mb
* @param schema
* @return
* @throws DMLRuntimeException
*/
@SuppressWarnings("resource")
private DataFrame createDataFrame(
SQLContext sqlctx, MatrixBlock mb, boolean containsID, ValueType[] schema)
throws DMLRuntimeException {
// create in-memory list of rows
List<Row> list = new ArrayList<Row>();
int off = (containsID ? 1 : 0);
int clen = mb.getNumColumns() + off - colsVector + 1;
for (int i = 0; i < mb.getNumRows(); i++) {
Object[] row = new Object[clen];
if (containsID) row[0] = i + 1;
for (int j = 0, j2 = 0; j < mb.getNumColumns(); j++, j2++) {
if (schema[j2] != ValueType.OBJECT) {
row[j2 + off] = UtilFunctions.doubleToObject(schema[j2], mb.quickGetValue(i, j));
} else {
double[] tmp =
DataConverter.convertToDoubleVector(
mb.sliceOperations(i, i, j, j + colsVector - 1, new MatrixBlock()));
row[j2 + off] = new DenseVector(tmp);
j += colsVector - 1;
}
}
list.add(RowFactory.create(row));
}
// create data frame schema
List<StructField> fields = new ArrayList<StructField>();
if (containsID)
fields.add(
DataTypes.createStructField(RDDConverterUtils.DF_ID_COLUMN, DataTypes.DoubleType, true));
for (int j = 0; j < schema.length; j++) {
DataType dt = null;
switch (schema[j]) {
case STRING:
dt = DataTypes.StringType;
break;
case DOUBLE:
dt = DataTypes.DoubleType;
break;
case INT:
dt = DataTypes.LongType;
break;
case OBJECT:
dt = new VectorUDT();
break;
default:
throw new RuntimeException("Unsupported value type.");
}
fields.add(DataTypes.createStructField("C" + (j + 1), dt, true));
}
StructType dfSchema = DataTypes.createStructType(fields);
// create rdd and data frame
JavaSparkContext sc = new JavaSparkContext(sqlctx.sparkContext());
JavaRDD<Row> rowRDD = sc.parallelize(list);
return sqlctx.createDataFrame(rowRDD, dfSchema);
}
/**
* Function to perform Eigen decomposition on a given matrix. Input must be a symmetric matrix.
*
* @param in matrix object
* @return array of matrix blocks
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock[] computeEigen(MatrixObject in) throws DMLRuntimeException {
if (in.getNumRows() != in.getNumColumns()) {
throw new DMLRuntimeException(
"Eigen Decomposition can only be done on a square matrix. Input matrix is rectangular (rows="
+ in.getNumRows()
+ ", cols="
+ in.getNumColumns()
+ ")");
}
Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);
EigenDecomposition eigendecompose = new EigenDecomposition(matrixInput);
RealMatrix eVectorsMatrix = eigendecompose.getV();
double[][] eVectors = eVectorsMatrix.getData();
double[] eValues = eigendecompose.getRealEigenvalues();
// Sort the eigen values (and vectors) in increasing order (to be compatible w/ LAPACK.DSYEVR())
int n = eValues.length;
for (int i = 0; i < n; i++) {
int k = i;
double p = eValues[i];
for (int j = i + 1; j < n; j++) {
if (eValues[j] < p) {
k = j;
p = eValues[j];
}
}
if (k != i) {
eValues[k] = eValues[i];
eValues[i] = p;
for (int j = 0; j < n; j++) {
p = eVectors[j][i];
eVectors[j][i] = eVectors[j][k];
eVectors[j][k] = p;
}
}
}
MatrixBlock mbValues = DataConverter.convertToMatrixBlock(eValues, true);
MatrixBlock mbVectors = DataConverter.convertToMatrixBlock(eVectors);
return new MatrixBlock[] {mbValues, mbVectors};
}
/** @param mb */
private void runTransposeSelfMatrixMultTest(
SparsityType sptype, ValueType vtype, boolean compress) {
try {
// prepare sparsity for input data
double sparsity = -1;
switch (sptype) {
case DENSE:
sparsity = sparsity1;
break;
case SPARSE:
sparsity = sparsity2;
break;
case EMPTY:
sparsity = sparsity3;
break;
}
// generate input data
double min = (vtype == ValueType.CONST) ? 10 : -10;
double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7);
if (vtype == ValueType.RAND_ROUND) input = TestUtils.round(input);
MatrixBlock mb = DataConverter.convertToMatrixBlock(input);
// compress given matrix block
CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb);
if (compress) cmb.compress();
// matrix-vector uncompressed
MatrixBlock ret1 = mb.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT);
// matrix-vector compressed
MatrixBlock ret2 = cmb.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT);
// compare result with input
double[][] d1 = DataConverter.convertToDoubleMatrix(ret1);
double[][] d2 = DataConverter.convertToDoubleMatrix(ret2);
TestUtils.compareMatrices(d1, d2, cols, cols, 0.0000001);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
/**
* Function to compute Cholesky decomposition of the given input matrix. The input must be a real
* symmetric positive-definite matrix.
*
* @param in commons-math3 Array2DRowRealMatrix
* @return matrix block
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock computeCholesky(Array2DRowRealMatrix in) throws DMLRuntimeException {
if (!in.isSquare())
throw new DMLRuntimeException(
"Input to cholesky() must be square matrix -- given: a "
+ in.getRowDimension()
+ "x"
+ in.getColumnDimension()
+ " matrix.");
CholeskyDecomposition cholesky = new CholeskyDecomposition(in);
RealMatrix rmL = cholesky.getL();
return DataConverter.convertToMatrixBlock(rmL.getData());
}
/**
* Function to compute matrix inverse via matrix decomposition.
*
* @param in commons-math3 Array2DRowRealMatrix
* @return matrix block
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
private static MatrixBlock computeMatrixInverse(Array2DRowRealMatrix in)
throws DMLRuntimeException {
if (!in.isSquare())
throw new DMLRuntimeException(
"Input to inv() must be square matrix -- given: a "
+ in.getRowDimension()
+ "x"
+ in.getColumnDimension()
+ " matrix.");
QRDecomposition qrdecompose = new QRDecomposition(in);
DecompositionSolver solver = qrdecompose.getSolver();
RealMatrix inverseMatrix = solver.getInverse();
return DataConverter.convertToMatrixBlock(inverseMatrix.getData());
}