/**
   * Function to perform LU decomposition on a given matrix.
   *
   * @param in matrix object
   * @return array of matrix blocks
   * @throws DMLRuntimeException if DMLRuntimeException occurs
   */
  private static MatrixBlock[] computeLU(MatrixObject in) throws DMLRuntimeException {
    if (in.getNumRows() != in.getNumColumns()) {
      throw new DMLRuntimeException(
          "LU Decomposition can only be done on a square matrix. Input matrix is rectangular (rows="
              + in.getNumRows()
              + ", cols="
              + in.getNumColumns()
              + ")");
    }

    Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);

    // Perform LUP decomposition
    LUDecomposition ludecompose = new LUDecomposition(matrixInput);
    RealMatrix P = ludecompose.getP();
    RealMatrix L = ludecompose.getL();
    RealMatrix U = ludecompose.getU();

    // Read the results into native format
    MatrixBlock mbP = DataConverter.convertToMatrixBlock(P.getData());
    MatrixBlock mbL = DataConverter.convertToMatrixBlock(L.getData());
    MatrixBlock mbU = DataConverter.convertToMatrixBlock(U.getData());

    return new MatrixBlock[] {mbP, mbL, mbU};
  }
  /**
   * Function to perform QR decomposition on a given matrix.
   *
   * @param in matrix object
   * @return array of matrix blocks
   * @throws DMLRuntimeException if DMLRuntimeException occurs
   */
  private static MatrixBlock[] computeQR(MatrixObject in) throws DMLRuntimeException {
    Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);

    // Perform QR decomposition
    QRDecomposition qrdecompose = new QRDecomposition(matrixInput);
    RealMatrix H = qrdecompose.getH();
    RealMatrix R = qrdecompose.getR();

    // Read the results into native format
    MatrixBlock mbH = DataConverter.convertToMatrixBlock(H.getData());
    MatrixBlock mbR = DataConverter.convertToMatrixBlock(R.getData());

    return new MatrixBlock[] {mbH, mbR};
  }
 // turns 3X1 matrix back into coordinate instance (uses t)
 private static Coordinate MatrixToCoordinateInstance(String system, RealMatrix V, double t) {
   double[][] vec = V.getData();
   double vecX = vec[0][0];
   double vecY = vec[0][1];
   double vecZ = vec[0][2];
   return new Coordinate(system, vecX, vecY, vecZ, t);
 }
 public RealMatrix makeZta(RealMatrix userFeature, RealMatrix articleFeature) {
   RealMatrix product = userFeature.multiply(articleFeature.transpose());
   double[][] productData = product.getData();
   double[] productVector = new double[36];
   int count = 0;
   for (int row = 0; row < 6; row++) {
     for (int col = 0; col < 6; col++) {
       productVector[count] = productData[row][col];
       count++;
     }
   }
   return MatrixUtils.createColumnRealMatrix(productVector);
 }
  /**
   * Function to compute Cholesky decomposition of the given input matrix. The input must be a real
   * symmetric positive-definite matrix.
   *
   * @param in commons-math3 Array2DRowRealMatrix
   * @return matrix block
   * @throws DMLRuntimeException if DMLRuntimeException occurs
   */
  private static MatrixBlock computeCholesky(Array2DRowRealMatrix in) throws DMLRuntimeException {
    if (!in.isSquare())
      throw new DMLRuntimeException(
          "Input to cholesky() must be square matrix -- given: a "
              + in.getRowDimension()
              + "x"
              + in.getColumnDimension()
              + " matrix.");

    CholeskyDecomposition cholesky = new CholeskyDecomposition(in);
    RealMatrix rmL = cholesky.getL();

    return DataConverter.convertToMatrixBlock(rmL.getData());
  }
  /**
   * Function to compute matrix inverse via matrix decomposition.
   *
   * @param in commons-math3 Array2DRowRealMatrix
   * @return matrix block
   * @throws DMLRuntimeException if DMLRuntimeException occurs
   */
  private static MatrixBlock computeMatrixInverse(Array2DRowRealMatrix in)
      throws DMLRuntimeException {
    if (!in.isSquare())
      throw new DMLRuntimeException(
          "Input to inv() must be square matrix -- given: a "
              + in.getRowDimension()
              + "x"
              + in.getColumnDimension()
              + " matrix.");

    QRDecomposition qrdecompose = new QRDecomposition(in);
    DecompositionSolver solver = qrdecompose.getSolver();
    RealMatrix inverseMatrix = solver.getInverse();

    return DataConverter.convertToMatrixBlock(inverseMatrix.getData());
  }
  /**
   * Function to solve a given system of equations.
   *
   * @param in1 matrix object 1
   * @param in2 matrix object 2
   * @return matrix block
   * @throws DMLRuntimeException if DMLRuntimeException occurs
   */
  private static MatrixBlock computeSolve(MatrixObject in1, MatrixObject in2)
      throws DMLRuntimeException {
    Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in1);
    Array2DRowRealMatrix vectorInput = DataConverter.convertToArray2DRowRealMatrix(in2);

    /*LUDecompositionImpl ludecompose = new LUDecompositionImpl(matrixInput);
    DecompositionSolver lusolver = ludecompose.getSolver();
    RealMatrix solutionMatrix = lusolver.solve(vectorInput);*/

    // Setup a solver based on QR Decomposition
    QRDecomposition qrdecompose = new QRDecomposition(matrixInput);
    DecompositionSolver solver = qrdecompose.getSolver();
    // Invoke solve
    RealMatrix solutionMatrix = solver.solve(vectorInput);

    return DataConverter.convertToMatrixBlock(solutionMatrix.getData());
  }
  /**
   * Function to perform Eigen decomposition on a given matrix. Input must be a symmetric matrix.
   *
   * @param in matrix object
   * @return array of matrix blocks
   * @throws DMLRuntimeException if DMLRuntimeException occurs
   */
  private static MatrixBlock[] computeEigen(MatrixObject in) throws DMLRuntimeException {
    if (in.getNumRows() != in.getNumColumns()) {
      throw new DMLRuntimeException(
          "Eigen Decomposition can only be done on a square matrix. Input matrix is rectangular (rows="
              + in.getNumRows()
              + ", cols="
              + in.getNumColumns()
              + ")");
    }

    Array2DRowRealMatrix matrixInput = DataConverter.convertToArray2DRowRealMatrix(in);

    EigenDecomposition eigendecompose = new EigenDecomposition(matrixInput);
    RealMatrix eVectorsMatrix = eigendecompose.getV();
    double[][] eVectors = eVectorsMatrix.getData();
    double[] eValues = eigendecompose.getRealEigenvalues();

    // Sort the eigen values (and vectors) in increasing order (to be compatible w/ LAPACK.DSYEVR())
    int n = eValues.length;
    for (int i = 0; i < n; i++) {
      int k = i;
      double p = eValues[i];
      for (int j = i + 1; j < n; j++) {
        if (eValues[j] < p) {
          k = j;
          p = eValues[j];
        }
      }
      if (k != i) {
        eValues[k] = eValues[i];
        eValues[i] = p;
        for (int j = 0; j < n; j++) {
          p = eVectors[j][i];
          eVectors[j][i] = eVectors[j][k];
          eVectors[j][k] = p;
        }
      }
    }

    MatrixBlock mbValues = DataConverter.convertToMatrixBlock(eValues, true);
    MatrixBlock mbVectors = DataConverter.convertToMatrixBlock(eVectors);

    return new MatrixBlock[] {mbValues, mbVectors};
  }
  @Override
  public double[][] correlate(
      List<DataSet> dataSets, Timestamp startDate, Timestamp endDate, TimeUnits granularity) {
    if (dataSets == null || dataSets.size() < 2) {
      throw new IllegalArgumentException("At least 2 data sets are required for correlation");
    }

    final Map<Long, List<Double>> orderedPoints = new TreeMap<Long, List<Double>>();

    for (int i = 0; i < dataSets.size(); i++) {
      final DataSet dataSet = dataSets.get(i);
      final List<DataSetPoint> points =
          this.dao.findAndAggregatePointsGroupedByUnit(dataSet, startDate, endDate, granularity);
      for (final DataSetPoint point : points) {
        final Long millis = point.getX().getTime();
        if (!orderedPoints.containsKey(millis)) {
          orderedPoints.put(millis, newDoubleList(dataSets.size()));
        }
        orderedPoints.get(millis).set(i, point.getY());
      }
    }

    if (orderedPoints.size() <= 1) {
      throw new IllegalArgumentException(
          "Needed at least 2 points, found: " + orderedPoints.size());
    }

    final double[][] points = new double[orderedPoints.size()][dataSets.size()];

    int i = 0;
    for (final Map.Entry<Long, List<Double>> entry : orderedPoints.entrySet()) {
      final List<Double> values = entry.getValue();
      for (int j = 0; j < values.size(); j++) {
        points[i][j] = values.get(j);
      }
      i++;
    }

    final PearsonsCorrelation correlation = new PearsonsCorrelation();
    final RealMatrix matrix = correlation.computeCorrelationMatrix(points);
    return matrix.getData();
  }
 /*  91:    */
 /*  92:    */ public Array2DRowRealMatrix initializeHighOrderDerivatives(
     double h, double[] t, double[][] y, double[][] yDot)
       /*  93:    */ {
   /*  94:259 */ double[][] a = new double[2 * (y.length - 1)][this.c1.length];
   /*  95:260 */ double[][] b = new double[2 * (y.length - 1)][y[0].length];
   /*  96:261 */ double[] y0 = y[0];
   /*  97:262 */ double[] yDot0 = yDot[0];
   /*  98:263 */ for (int i = 1; i < y.length; i++)
   /*  99:    */ {
     /* 100:265 */ double di = t[i] - t[0];
     /* 101:266 */ double ratio = di / h;
     /* 102:267 */ double dikM1Ohk = 1.0D / h;
     /* 103:    */
     /* 104:    */
     /* 105:    */
     /* 106:271 */ double[] aI = a[(2 * i - 2)];
     /* 107:272 */ double[] aDotI = a[(2 * i - 1)];
     /* 108:273 */ for (int j = 0; j < aI.length; j++)
     /* 109:    */ {
       /* 110:274 */ dikM1Ohk *= ratio;
       /* 111:275 */ aI[j] = (di * dikM1Ohk);
       /* 112:276 */ aDotI[j] = ((j + 2) * dikM1Ohk);
       /* 113:    */ }
     /* 114:280 */ double[] yI = y[i];
     /* 115:281 */ double[] yDotI = yDot[i];
     /* 116:282 */ double[] bI = b[(2 * i - 2)];
     /* 117:283 */ double[] bDotI = b[(2 * i - 1)];
     /* 118:284 */ for (int j = 0; j < yI.length; j++)
     /* 119:    */ {
       /* 120:285 */ bI[j] = (yI[j] - y0[j] - di * yDot0[j]);
       /* 121:286 */ yDotI[j] -= yDot0[j];
       /* 122:    */ }
     /* 123:    */ }
   /* 124:294 */ QRDecomposition decomposition =
       new QRDecomposition(new Array2DRowRealMatrix(a, false));
   /* 125:295 */ RealMatrix x =
       decomposition.getSolver().solve(new Array2DRowRealMatrix(b, false));
   /* 126:296 */ return new Array2DRowRealMatrix(x.getData(), false);
   /* 127:    */ }
示例#11
0
  @Override
  public Article chooseArm(User user, List<Article> articles) {
    Article bestA = null;
    double bestArmP = Double.MIN_VALUE;

    RealMatrix Aa;
    RealMatrix Ba;
    RealMatrix ba;

    for (Article a : articles) {
      String aId = a.getId();
      if (!AMap.containsKey(aId)) {
        Aa = MatrixUtils.createRealIdentityMatrix(6);
        AMap.put(aId, Aa); // set as identity for now and we will update
        // in reward

        double[] zeros = {0, 0, 0, 0, 0, 0};
        ba = MatrixUtils.createColumnRealMatrix(zeros);
        bMap.put(aId, ba);

        double[][] BMapZeros = new double[6][36];
        for (double[] row : BMapZeros) {
          Arrays.fill(row, 0.0);
        }
        Ba = MatrixUtils.createRealMatrix(BMapZeros);
        BMap.put(aId, Ba);
      } else {
        Aa = AMap.get(aId);
        ba = bMap.get(aId);
        Ba = BMap.get(aId);
      }

      // Make column vector out of features
      RealMatrix xta = MatrixUtils.createColumnRealMatrix(a.getFeatures());
      RealMatrix zta = makeZta(MatrixUtils.createColumnRealMatrix(user.getFeatures()), xta);

      // Set up common variables
      RealMatrix A0Inverse = MatrixUtils.inverse(A0);
      RealMatrix AaInverse = MatrixUtils.inverse(Aa);
      RealMatrix ztaTranspose = zta.transpose();
      RealMatrix BaTranspose = Ba.transpose();
      RealMatrix xtaTranspose = xta.transpose();

      // Find theta
      RealMatrix theta = AaInverse.multiply(ba.subtract(Ba.multiply(BetaHat)));
      // Find sta
      RealMatrix staMatrix = ztaTranspose.multiply(A0Inverse).multiply(zta);
      staMatrix =
          staMatrix.subtract(
              ztaTranspose
                  .multiply(A0Inverse)
                  .multiply(BaTranspose)
                  .multiply(AaInverse)
                  .multiply(xta)
                  .scalarMultiply(2));
      staMatrix = staMatrix.add(xtaTranspose.multiply(AaInverse).multiply(xta));
      staMatrix =
          staMatrix.add(
              xtaTranspose
                  .multiply(AaInverse)
                  .multiply(Ba)
                  .multiply(A0Inverse)
                  .multiply(BaTranspose)
                  .multiply(AaInverse)
                  .multiply(xta));

      // Find pta for arm
      RealMatrix ptaMatrix = ztaTranspose.multiply(BetaHat);
      ptaMatrix = ptaMatrix.add(xtaTranspose.multiply(theta));
      double ptaVal = ptaMatrix.getData()[0][0];
      double staVal = staMatrix.getData()[0][0];
      ptaVal = ptaVal + alpha * Math.sqrt(staVal);

      // Update argmax
      if (ptaVal > bestArmP) {
        bestArmP = ptaVal;
        bestA = a;
      }
    }
    return bestA;
  }
示例#12
0
  /**
   * Tangent normalize a coverage profile.
   *
   * <p>Notes about the Spark tangent normalization can be found in docs/PoN/
   *
   * @param pon Not {@code null}
   * @param targetFactorNormalizedCounts ReadCountCollection of counts that have already been
   *     normalized fully (typically, including the target factor normalization). I.e. a coverage
   *     profile The column names should be intact. Not {@code null} See {@link
   *     TangentNormalizer::createCoverageProfile}
   * @return never {@code null}
   */
  private static TangentNormalizationResult tangentNormalize(
      final PoN pon, final ReadCountCollection targetFactorNormalizedCounts, JavaSparkContext ctx) {

    Utils.nonNull(pon, "PoN cannot be null.");
    Utils.nonNull(targetFactorNormalizedCounts, "targetFactorNormalizedCounts cannot be null.");
    Utils.nonNull(
        targetFactorNormalizedCounts.columnNames(),
        "targetFactorNormalizedCounts column names cannot be null.");
    ParamUtils.isPositive(
        targetFactorNormalizedCounts.columnNames().size(),
        "targetFactorNormalizedCounts column names cannot be an empty list.");

    final Case2PoNTargetMapper targetMapper =
        new Case2PoNTargetMapper(targetFactorNormalizedCounts.targets(), pon.getPanelTargetNames());

    // The input counts with rows (targets) sorted so that they match the PoN's order.
    final RealMatrix tangentNormalizationRawInputCounts =
        targetMapper.fromCaseToPoNCounts(targetFactorNormalizedCounts.counts());

    // We prepare the counts for tangent normalization.
    final RealMatrix tangentNormalizationInputCounts =
        composeTangentNormalizationInputMatrix(tangentNormalizationRawInputCounts);

    if (ctx == null) {

      // Calculate the beta-hats for the input read count columns (samples).
      logger.info("Calculating beta hats...");
      final RealMatrix tangentBetaHats =
          pon.betaHats(tangentNormalizationInputCounts, true, EPSILON);

      // Actual tangent normalization step.
      logger.info(
          "Performing actual tangent normalization ("
              + tangentNormalizationInputCounts.getColumnDimension()
              + " columns)...");
      final RealMatrix tangentNormalizedCounts =
          pon.tangentNormalization(tangentNormalizationInputCounts, tangentBetaHats, true);

      // Output the tangent normalized counts.
      logger.info("Post-processing tangent normalization results...");
      final ReadCountCollection tangentNormalized =
          targetMapper.fromPoNtoCaseCountCollection(
              tangentNormalizedCounts, targetFactorNormalizedCounts.columnNames());
      final ReadCountCollection preTangentNormalized =
          targetMapper.fromPoNtoCaseCountCollection(
              tangentNormalizationInputCounts, targetFactorNormalizedCounts.columnNames());

      return new TangentNormalizationResult(
          tangentNormalized, preTangentNormalized, tangentBetaHats, targetFactorNormalizedCounts);

    } else {

      /*
      Using Spark:  the code here is a little more complex for optimization purposes.

      Please see notes in docs/PoN ...

      Ahat^T = (C^T P^T) A^T
      Therefore, C^T is the RowMatrix

      pinv: P
      panel: A
      projection: Ahat
      cases: C
      betahat: C^T P^T
      tangentNormalizedCounts: C - Ahat
       */
      final RealMatrix pinv = pon.getReducedPanelPInverseCounts();
      final RealMatrix panel = pon.getReducedPanelCounts();

      // Make the C^T a distributed matrix (RowMatrix)
      final RowMatrix caseTDistMat =
          SparkConverter.convertRealMatrixToSparkRowMatrix(
              ctx, tangentNormalizationInputCounts.transpose(), TN_NUM_SLICES_SPARK);

      // Spark local matrices (transposed)
      final Matrix pinvTLocalMat =
          new DenseMatrix(
                  pinv.getRowDimension(),
                  pinv.getColumnDimension(),
                  Doubles.concat(pinv.getData()),
                  true)
              .transpose();
      final Matrix panelTLocalMat =
          new DenseMatrix(
                  panel.getRowDimension(),
                  panel.getColumnDimension(),
                  Doubles.concat(panel.getData()),
                  true)
              .transpose();

      // Calculate the projection transpose in a distributed matrix, then convert to Apache Commons
      // matrix (not transposed)
      final RowMatrix betahatDistMat = caseTDistMat.multiply(pinvTLocalMat);
      final RowMatrix projectionTDistMat = betahatDistMat.multiply(panelTLocalMat);
      final RealMatrix projection =
          SparkConverter.convertSparkRowMatrixToRealMatrix(
                  projectionTDistMat, tangentNormalizationInputCounts.transpose().getRowDimension())
              .transpose();

      // Subtract the cases from the projection
      final RealMatrix tangentNormalizedCounts =
          tangentNormalizationInputCounts.subtract(projection);

      // Construct the result object and return it with the correct targets.
      final ReadCountCollection tangentNormalized =
          targetMapper.fromPoNtoCaseCountCollection(
              tangentNormalizedCounts, targetFactorNormalizedCounts.columnNames());
      final ReadCountCollection preTangentNormalized =
          targetMapper.fromPoNtoCaseCountCollection(
              tangentNormalizationInputCounts, targetFactorNormalizedCounts.columnNames());
      final RealMatrix tangentBetaHats =
          SparkConverter.convertSparkRowMatrixToRealMatrix(
              betahatDistMat, tangentNormalizedCounts.getColumnDimension());
      return new TangentNormalizationResult(
          tangentNormalized,
          preTangentNormalized,
          tangentBetaHats.transpose(),
          targetFactorNormalizedCounts);
    }
  }
示例#13
0
  private double generalizedCorrelationRatio(SampleIterator it, int inputDim, int out) {
    Map<Double, Integer> n_y = new HashMap<>();
    Map<Double, MultivariateSummaryStatistics> stat_y = new HashMap<>();
    List<RealMatrix> x = new ArrayList<>();
    MultivariateSummaryStatistics stat = new MultivariateSummaryStatistics(inputDim, unbiased);

    for (int i = 0; i < maxSamples && it.hasNext(); i++) {
      Sample sample = it.next();
      double[] input = sample.getEncodedInput().toArray();
      double output = sample.getEncodedOutput().getEntry(out);
      if (!n_y.containsKey(output)) {
        n_y.put(output, 0);
        stat_y.put(output, new MultivariateSummaryStatistics(inputDim, unbiased));
      }

      injectNoise(input);
      n_y.put(output, n_y.get(output) + 1);
      stat_y.get(output).addValue(input);
      x.add(new Array2DRowRealMatrix(input));
      stat.addValue(input);
    }

    RealMatrix x_sum = new Array2DRowRealMatrix(stat.getSum());
    Map<Double, RealMatrix> x_y_sum = new HashMap<>();
    for (Entry<Double, MultivariateSummaryStatistics> entry : stat_y.entrySet()) {
      x_y_sum.put(entry.getKey(), new Array2DRowRealMatrix(entry.getValue().getSum()));
    }

    RealMatrix H = new Array2DRowRealMatrix(inputDim, inputDim);
    RealMatrix temp = new Array2DRowRealMatrix(inputDim, inputDim);

    for (double key : n_y.keySet()) {
      temp =
          temp.add(
              x_y_sum
                  .get(key)
                  .multiply(x_y_sum.get(key).transpose())
                  .scalarMultiply(1.0 / n_y.get(key)));
    }
    H = temp.subtract(x_sum.multiply(x_sum.transpose()).scalarMultiply(1.0 / x.size()));

    RealMatrix E = new Array2DRowRealMatrix(inputDim, inputDim);
    for (RealMatrix m : x) {
      E = E.add(m.multiply(m.transpose()));
    }
    E = E.subtract(temp);

    List<Integer> zeroColumns = findZeroColumns(E);
    E = removeZeroColumns(E, zeroColumns);
    H = removeZeroColumns(H, zeroColumns);

    Matrix JE = new Matrix(E.getData());
    Matrix JH = new Matrix(H.getData());

    if (JE.rank() < JE.getRowDimension()) {
      Log.write(this, "Some error occurred (E matrix is singular)");
      return -1;
    } else {
      double lambda;
      if (useEigenvalues) {
        Matrix L = JE.inverse().times(JH);
        double[] eigs = L.eig().getRealEigenvalues();
        Arrays.sort(eigs);

        lambda = 1;
        int nonNullEigs = n_y.keySet().size() - 1;
        for (int i = eigs.length - nonNullEigs; i < eigs.length; i++) {
          if (Math.abs(eigs[i]) < zeroThreshold) {
            Log.write(this, "Some error occurred (E matrix has too many null eigenvalues)");
            return -1;
          }
          lambda *= 1.0 / (1.0 + eigs[i]);
        }
      } else {
        Matrix sum = JE.plus(JH);
        if (sum.rank() < sum.getRowDimension()) {
          Log.write(this, "Some error occourred (E+H is singular");
          return -1;
        }
        lambda = JE.det() / sum.det();
      }

      return Math.sqrt(1 - lambda);
    }
  }