/** * Function to perform LU decomposition on a given matrix. * * @param in matrix object * @return array of matrix blocks * @throws DMLRuntimeException if DMLRuntimeException occurs */ private static MatrixBlock[] computeLU(MatrixObject in) throws DMLRuntimeException { if (in.getNumRows() != in.getNumColumns()) { throw new DMLRuntimeException( "LU Decomposition can only be done on a square matrix. Input matrix is rectangular (rows=" + in.getNumRows() + ", cols=" + in.getNumColumns() + ")"); } Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in); // Perform LUP decomposition LUDecomposition ludecompose = new LUDecomposition(matrixInput); RealMatrix P = ludecompose.getP(); RealMatrix L = ludecompose.getL(); RealMatrix U = ludecompose.getU(); // Read the results into native format MatrixBlock mbP = DataConverter.convertToMatrixBlock(P.getData()); MatrixBlock mbL = DataConverter.convertToMatrixBlock(L.getData()); MatrixBlock mbU = DataConverter.convertToMatrixBlock(U.getData()); return new MatrixBlock[] {mbP, mbL, mbU}; }
/** * @param vector * @param singleColBlock * @param dense * @param unknownDims */ private void testDataFrameConversion( ValueType[] schema, boolean containsID, boolean dense, boolean unknownDims) { boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG; RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform; SparkExecutionContext sec = null; try { DMLScript.USE_LOCAL_SPARK_CONFIG = true; DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; // generate input data and setup metadata int cols = schema.length + colsVector - 1; double sparsity = dense ? sparsity1 : sparsity2; double[][] A = TestUtils.round(getRandomMatrix(rows1, cols, -10, 1000, sparsity, 2373)); MatrixBlock mbA = DataConverter.convertToMatrixBlock(A); int blksz = ConfigurationManager.getBlocksize(); MatrixCharacteristics mc1 = new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros()); MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1); // setup spark context sec = (SparkExecutionContext) ExecutionContextFactory.createContext(); JavaSparkContext sc = sec.getSparkContext(); SQLContext sqlctx = new SQLContext(sc); // create input data frame DataFrame df = createDataFrame(sqlctx, mbA, containsID, schema); // dataframe - frame conversion JavaPairRDD<Long, FrameBlock> out = FrameRDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc2, containsID); // get output frame block FrameBlock fbB = SparkExecutionContext.toFrameBlock( out, UtilFunctions.nCopies(cols, ValueType.DOUBLE), rows1, cols); // compare frame blocks MatrixBlock mbB = DataConverter.convertToMatrixBlock(fbB); double[][] B = DataConverter.convertToDoubleMatrix(mbB); TestUtils.compareMatrices(A, B, rows1, cols, eps); } catch (Exception ex) { throw new RuntimeException(ex); } finally { sec.close(); DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig; DMLScript.rtplatform = oldPlatform; } }
/** * Function to perform QR decomposition on a given matrix. * * @param in matrix object * @return array of matrix blocks * @throws DMLRuntimeException if DMLRuntimeException occurs */ private static MatrixBlock[] computeQR(MatrixObject in) throws DMLRuntimeException { Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in); // Perform QR decomposition QRDecomposition qrdecompose = new QRDecomposition(matrixInput); RealMatrix H = qrdecompose.getH(); RealMatrix R = qrdecompose.getR(); // Read the results into native format MatrixBlock mbH = DataConverter.convertToMatrixBlock(H.getData()); MatrixBlock mbR = DataConverter.convertToMatrixBlock(R.getData()); return new MatrixBlock[] {mbH, mbR}; }
public static MatrixBlock unaryOperations(MatrixObject inj, String opcode) throws DMLRuntimeException { Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(inj); if (opcode.equals("inverse")) return computeMatrixInverse(matrixInput); else if (opcode.equals("cholesky")) return computeCholesky(matrixInput); return null; }
/** * Function to solve a given system of equations. * * @param in1 matrix object 1 * @param in2 matrix object 2 * @return matrix block * @throws DMLRuntimeException if DMLRuntimeException occurs */ private static MatrixBlock computeSolve(MatrixObject in1, MatrixObject in2) throws DMLRuntimeException { Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in1); Array2DRowRealMatrix vectorInput = DataConverter.convertToArray2DRowRealMatrix(in2); /*LUDecompositionImpl ludecompose = new LUDecompositionImpl(matrixInput); DecompositionSolver lusolver = ludecompose.getSolver(); RealMatrix solutionMatrix = lusolver.solve(vectorInput);*/ // Setup a solver based on QR Decomposition QRDecomposition qrdecompose = new QRDecomposition(matrixInput); DecompositionSolver solver = qrdecompose.getSolver(); // Invoke solve RealMatrix solutionMatrix = solver.solve(vectorInput); return DataConverter.convertToMatrixBlock(solutionMatrix.getData()); }
/** * @param sqlctx * @param mb * @param schema * @return * @throws DMLRuntimeException */ @SuppressWarnings("resource") private DataFrame createDataFrame( SQLContext sqlctx, MatrixBlock mb, boolean containsID, ValueType[] schema) throws DMLRuntimeException { // create in-memory list of rows List<Row> list = new ArrayList<Row>(); int off = (containsID ? 1 : 0); int clen = mb.getNumColumns() + off - colsVector + 1; for (int i = 0; i < mb.getNumRows(); i++) { Object[] row = new Object[clen]; if (containsID) row[0] = i + 1; for (int j = 0, j2 = 0; j < mb.getNumColumns(); j++, j2++) { if (schema[j2] != ValueType.OBJECT) { row[j2 + off] = UtilFunctions.doubleToObject(schema[j2], mb.quickGetValue(i, j)); } else { double[] tmp = DataConverter.convertToDoubleVector( mb.sliceOperations(i, i, j, j + colsVector - 1, new MatrixBlock())); row[j2 + off] = new DenseVector(tmp); j += colsVector - 1; } } list.add(RowFactory.create(row)); } // create data frame schema List<StructField> fields = new ArrayList<StructField>(); if (containsID) fields.add( DataTypes.createStructField(RDDConverterUtils.DF_ID_COLUMN, DataTypes.DoubleType, true)); for (int j = 0; j < schema.length; j++) { DataType dt = null; switch (schema[j]) { case STRING: dt = DataTypes.StringType; break; case DOUBLE: dt = DataTypes.DoubleType; break; case INT: dt = DataTypes.LongType; break; case OBJECT: dt = new VectorUDT(); break; default: throw new RuntimeException("Unsupported value type."); } fields.add(DataTypes.createStructField("C" + (j + 1), dt, true)); } StructType dfSchema = DataTypes.createStructType(fields); // create rdd and data frame JavaSparkContext sc = new JavaSparkContext(sqlctx.sparkContext()); JavaRDD<Row> rowRDD = sc.parallelize(list); return sqlctx.createDataFrame(rowRDD, dfSchema); }
/** * Function to perform Eigen decomposition on a given matrix. Input must be a symmetric matrix. * * @param in matrix object * @return array of matrix blocks * @throws DMLRuntimeException if DMLRuntimeException occurs */ private static MatrixBlock[] computeEigen(MatrixObject in) throws DMLRuntimeException { if (in.getNumRows() != in.getNumColumns()) { throw new DMLRuntimeException( "Eigen Decomposition can only be done on a square matrix. Input matrix is rectangular (rows=" + in.getNumRows() + ", cols=" + in.getNumColumns() + ")"); } Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in); EigenDecomposition eigendecompose = new EigenDecomposition(matrixInput); RealMatrix eVectorsMatrix = eigendecompose.getV(); double[][] eVectors = eVectorsMatrix.getData(); double[] eValues = eigendecompose.getRealEigenvalues(); // Sort the eigen values (and vectors) in increasing order (to be compatible w/ LAPACK.DSYEVR()) int n = eValues.length; for (int i = 0; i < n; i++) { int k = i; double p = eValues[i]; for (int j = i + 1; j < n; j++) { if (eValues[j] < p) { k = j; p = eValues[j]; } } if (k != i) { eValues[k] = eValues[i]; eValues[i] = p; for (int j = 0; j < n; j++) { p = eVectors[j][i]; eVectors[j][i] = eVectors[j][k]; eVectors[j][k] = p; } } } MatrixBlock mbValues = DataConverter.convertToMatrixBlock(eValues, true); MatrixBlock mbVectors = DataConverter.convertToMatrixBlock(eVectors); return new MatrixBlock[] {mbValues, mbVectors}; }
/** @param mb */ private void runTransposeSelfMatrixMultTest( SparsityType sptype, ValueType vtype, boolean compress) { try { // prepare sparsity for input data double sparsity = -1; switch (sptype) { case DENSE: sparsity = sparsity1; break; case SPARSE: sparsity = sparsity2; break; case EMPTY: sparsity = sparsity3; break; } // generate input data double min = (vtype == ValueType.CONST) ? 10 : -10; double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7); if (vtype == ValueType.RAND_ROUND) input = TestUtils.round(input); MatrixBlock mb = DataConverter.convertToMatrixBlock(input); // compress given matrix block CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb); if (compress) cmb.compress(); // matrix-vector uncompressed MatrixBlock ret1 = mb.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT); // matrix-vector compressed MatrixBlock ret2 = cmb.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT); // compare result with input double[][] d1 = DataConverter.convertToDoubleMatrix(ret1); double[][] d2 = DataConverter.convertToDoubleMatrix(ret2); TestUtils.compareMatrices(d1, d2, cols, cols, 0.0000001); } catch (Exception ex) { throw new RuntimeException(ex); } }
/** * Function to compute Cholesky decomposition of the given input matrix. The input must be a real * symmetric positive-definite matrix. * * @param in commons-math3 Array2DRowRealMatrix * @return matrix block * @throws DMLRuntimeException if DMLRuntimeException occurs */ private static MatrixBlock computeCholesky(Array2DRowRealMatrix in) throws DMLRuntimeException { if (!in.isSquare()) throw new DMLRuntimeException( "Input to cholesky() must be square matrix -- given: a " + in.getRowDimension() + "x" + in.getColumnDimension() + " matrix."); CholeskyDecomposition cholesky = new CholeskyDecomposition(in); RealMatrix rmL = cholesky.getL(); return DataConverter.convertToMatrixBlock(rmL.getData()); }
/** * Function to compute matrix inverse via matrix decomposition. * * @param in commons-math3 Array2DRowRealMatrix * @return matrix block * @throws DMLRuntimeException if DMLRuntimeException occurs */ private static MatrixBlock computeMatrixInverse(Array2DRowRealMatrix in) throws DMLRuntimeException { if (!in.isSquare()) throw new DMLRuntimeException( "Input to inv() must be square matrix -- given: a " + in.getRowDimension() + "x" + in.getColumnDimension() + " matrix."); QRDecomposition qrdecompose = new QRDecomposition(in); DecompositionSolver solver = qrdecompose.getSolver(); RealMatrix inverseMatrix = solver.getInverse(); return DataConverter.convertToMatrixBlock(inverseMatrix.getData()); }