コード例 #1
0
  // Makes and scales the matrices V, D, and VT (to avoid ugly decimals)
  private void makeVDVT(EigenDecomp ed) {
    V = ed.getV();
    D = ed.getD();
    VT = ed.getVT();
    double ref = 0;

    for (int i = 0; i < V.getRowDimension(); i++) {
      ref = 0;
      for (int j = 0; j < V.getColumnDimension(); j++) {
        if (V.getEntry(j, i) != 0 && ref == 0) {
          ref = V.getEntry(j, i);
        }
        if (ref != 0) {
          V.setEntry(j, i, V.getEntry(j, i) / Math.abs(ref));
        }
      }
    }

    for (int i = 0; i < VT.getRowDimension(); i++) {
      ref = 0;
      for (int j = 0; j < VT.getColumnDimension(); j++) {
        if (VT.getEntry(j, i) != 0 && ref == 0) {
          ref = VT.getEntry(j, i);
        }
        if (ref != 0) {
          VT.setEntry(j, i, VT.getEntry(j, i) / Math.abs(ref));
        }
      }
    }
  }
  /**
   * Build the SVD model.
   *
   * @return A singular value decomposition recommender model.
   */
  @Override
  public SVDModel get() {
    // Create index mappings of user and item IDs.
    // You can use these to find row and columns in the matrix based on user/item IDs.
    IdIndexMapping userMapping = IdIndexMapping.create(userDAO.getUserIds());
    logger.debug("indexed {} users", userMapping.size());
    IdIndexMapping itemMapping = IdIndexMapping.create(itemDAO.getItemIds());
    logger.debug("indexed {} items", itemMapping.size());

    // We have to do 2 things:
    // First, prepare a matrix containing the rating data.
    RealMatrix matrix = createRatingMatrix(userMapping, itemMapping);

    // Second, compute its factorization
    // All the work is done in the constructor
    SingularValueDecomposition svd = new SingularValueDecomposition(matrix);

    // Third, truncate the decomposed matrix
    // TODO Truncate the matrices and construct the SVD model
    RealMatrix userMatrix = svd.getU();
    RealMatrix weights = svd.getS();
    RealMatrix itemMatrix = svd.getV();

    userMatrix = userMatrix.getSubMatrix(0, userMatrix.getRowDimension() - 1, 0, featureCount - 1);
    weights = weights.getSubMatrix(0, featureCount - 1, 0, featureCount - 1);
    itemMatrix = itemMatrix.getSubMatrix(0, itemMatrix.getRowDimension() - 1, 0, featureCount - 1);

    return new SVDModel(userMapping, itemMapping, userMatrix, itemMatrix, weights);
  }
コード例 #3
0
ファイル: MatrixTest.java プロジェクト: theseusyang/graphulo
  @Test
  public void testInverseIdentity() {
    double tol = 0.00001;
    Map<Key, Value> input = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
    input.put(new Key("1", "", "1"), new Value("1".getBytes()));
    //    input.put(new Key("1", "", "2"), new Value("1".getBytes()));
    //    input.put(new Key("2", "", "1"), new Value("1".getBytes()));
    input.put(new Key("2", "", "2"), new Value("1".getBytes()));

    RealMatrix matrix = MemMatrixUtil.buildMatrix(input.entrySet().iterator(), 2);
    Assert.assertEquals(2, matrix.getRowDimension());
    Assert.assertEquals(2, matrix.getColumnDimension());
    Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
    Assert.assertEquals(0, matrix.getEntry(0, 1), tol);
    Assert.assertEquals(0, matrix.getEntry(1, 0), tol);
    Assert.assertEquals(1, matrix.getEntry(1, 1), tol);

    matrix = MemMatrixUtil.doInverse(matrix, -1);
    Assert.assertEquals(2, matrix.getRowDimension());
    Assert.assertEquals(2, matrix.getColumnDimension());
    Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
    Assert.assertEquals(0, matrix.getEntry(0, 1), tol);
    Assert.assertEquals(0, matrix.getEntry(1, 0), tol);
    Assert.assertEquals(1, matrix.getEntry(1, 1), tol);

    SortedMap<Key, Value> back =
        MemMatrixUtil.matrixToMap(new TreeMap<Key, Value>(TestUtil.COMPARE_KEY_TO_COLQ), matrix);
    TestUtil.assertEqualDoubleMap(input, back);
    //    Assert.assertEquals(1, Double.parseDouble(new String(back.get(new Key("1", "",
    // "1")).get())), tol);
    //    Assert.assertEquals(1, Double.parseDouble(new String(back.get(new Key("2", "",
    // "2")).get())), tol);
  }
コード例 #4
0
ファイル: Weight.java プロジェクト: quedexco/idylfin
  /**
   * @param weight Weight matrix.
   * @throws NonSquareMatrixException if the argument is not a square matrix.
   */
  public Weight(RealMatrix weight) {
    if (weight.getColumnDimension() != weight.getRowDimension()) {
      throw new NonSquareMatrixException(weight.getColumnDimension(), weight.getRowDimension());
    }

    weightMatrix = weight.copy();
  }
コード例 #5
0
ファイル: MatrixTest.java プロジェクト: theseusyang/graphulo
  @Test
  public void testInverse2x2() {
    double tol = 0.001;
    Map<Key, Value> input = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
    input.put(new Key("1", "", "1"), new Value("4".getBytes()));
    input.put(new Key("1", "", "2"), new Value("3".getBytes()));
    input.put(new Key("2", "", "1"), new Value("1".getBytes()));
    input.put(new Key("2", "", "2"), new Value("1".getBytes()));
    Map<Key, Value> expect = new TreeMap<>(TestUtil.COMPARE_KEY_TO_COLQ);
    expect.put(new Key("1", "", "1"), new Value("1 ".getBytes()));
    expect.put(new Key("1", "", "2"), new Value("-3".getBytes()));
    expect.put(new Key("2", "", "1"), new Value("-1".getBytes()));
    expect.put(new Key("2", "", "2"), new Value("4 ".getBytes()));

    RealMatrix matrix = MemMatrixUtil.buildMatrix(input.entrySet().iterator(), 2);
    Assert.assertEquals(2, matrix.getRowDimension());
    Assert.assertEquals(2, matrix.getColumnDimension());
    Assert.assertEquals(4, matrix.getEntry(0, 0), tol);
    Assert.assertEquals(3, matrix.getEntry(0, 1), tol);
    Assert.assertEquals(1, matrix.getEntry(1, 0), tol);
    Assert.assertEquals(1, matrix.getEntry(1, 1), tol);

    matrix = MemMatrixUtil.doInverse(matrix, -1);
    Assert.assertEquals(2, matrix.getRowDimension());
    Assert.assertEquals(2, matrix.getColumnDimension());
    Assert.assertEquals(1, matrix.getEntry(0, 0), tol);
    Assert.assertEquals(-3, matrix.getEntry(0, 1), tol);
    Assert.assertEquals(-1, matrix.getEntry(1, 0), tol);
    Assert.assertEquals(4, matrix.getEntry(1, 1), tol);

    SortedMap<Key, Value> back =
        MemMatrixUtil.matrixToMap(new TreeMap<Key, Value>(TestUtil.COMPARE_KEY_TO_COLQ), matrix);
    TestUtil.assertEqualDoubleMap(expect, back);
  }
コード例 #6
0
  public double computeSimilarity(RealMatrix sourceDoc, RealMatrix targetDoc) {
    if (sourceDoc.getRowDimension() != targetDoc.getRowDimension()
        || sourceDoc.getColumnDimension() != targetDoc.getColumnDimension()
        || sourceDoc.getColumnDimension() != 1) {
      throw new IllegalArgumentException(
          "Matrices are not column matrices or not of the same size");
    }
    double[] source = sourceDoc.getColumn(0);
    double[] target = targetDoc.getColumn(0);

    double dotProduct = dot(source, target);
    double distance = norm(source) * norm(target);
    return dotProduct / distance;
  }
コード例 #7
0
 private RealMatrix normalizeData(RealMatrix matrix, UserProfileEigenModel model) {
   RealMatrix normalizedData =
       new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension());
   if (LOG.isDebugEnabled()) LOG.debug("model statistics size: " + model.statistics().length);
   for (int i = 0; i < matrix.getRowDimension(); i++) {
     for (int j = 0; j < matrix.getColumnDimension(); j++) {
       double value =
           (matrix.getEntry(i, j) - model.statistics()[j].getMean())
               / model.statistics()[j].getStddev();
       normalizedData.setEntry(i, j, value);
     }
   }
   return normalizedData;
 }
コード例 #8
0
  /**
   * Applies rank transform to each of the columns of <code>matrix</code> using the current <code>
   * rankingAlgorithm</code>
   *
   * @param matrix matrix to transform
   * @return a rank-transformed matrix
   */
  private RealMatrix rankTransform(final RealMatrix matrix) {
    RealMatrix transformed = null;

    if (rankingAlgorithm instanceof NaturalRanking
        && ((NaturalRanking) rankingAlgorithm).getNanStrategy() == NaNStrategy.REMOVED) {
      final Set<Integer> nanPositions = new HashSet<Integer>();
      for (int i = 0; i < matrix.getColumnDimension(); i++) {
        nanPositions.addAll(getNaNPositions(matrix.getColumn(i)));
      }

      // if we have found NaN values, we have to update the matrix size
      if (!nanPositions.isEmpty()) {
        transformed =
            new BlockRealMatrix(
                matrix.getRowDimension() - nanPositions.size(), matrix.getColumnDimension());
        for (int i = 0; i < transformed.getColumnDimension(); i++) {
          transformed.setColumn(i, removeValues(matrix.getColumn(i), nanPositions));
        }
      }
    }

    if (transformed == null) {
      transformed = matrix.copy();
    }

    for (int i = 0; i < transformed.getColumnDimension(); i++) {
      transformed.setColumn(i, rankingAlgorithm.rank(transformed.getColumn(i)));
    }

    return transformed;
  }
コード例 #9
0
ファイル: CorrelationRatio.java プロジェクト: etamponi/game
 private RealMatrix removeZeroColumns(RealMatrix base, List<Integer> zeroColumns) {
   int adjustedDim = base.getRowDimension() - zeroColumns.size();
   if (adjustedDim == 0) return base;
   RealMatrix adjusted = new Array2DRowRealMatrix(adjustedDim, adjustedDim);
   int i = 0, j = 0;
   for (int basei = 0; basei < base.getRowDimension(); basei++) {
     if (zeroColumns.contains(basei)) continue;
     for (int basej = 0; basej < base.getColumnDimension(); basej++) {
       if (zeroColumns.contains(basej)) continue;
       adjusted.setEntry(i, j++, base.getEntry(basei, basej));
     }
     i++;
     j = 0;
   }
   return adjusted;
 }
コード例 #10
0
ファイル: otherTools.java プロジェクト: Pattrick20/DLcty
  public static RealMatrix stochasticSubmatrix(RealMatrix data, int batch_size, Random rng) {
    // assume all data has the size number_samples by number_features
    int num_samples = data.getRowDimension();
    int num_features = data.getColumnDimension();
    int batch_num = num_samples / batch_size + 1;

    // randomly generate a batch index
    int batch_index = rng.nextInt(batch_num);
    List<Integer> rowIndex_tmp = new ArrayList<Integer>();

    for (int i = 0; i < batch_size; i++) {
      if (batch_size * batch_index + i >= num_samples) {
        break;
      } else {
        rowIndex_tmp.add(batch_size * batch_index + i);
      }
    }
    int[] rowIndex = TypeConvert.ArrayTointv(rowIndex_tmp);

    // System.out.println(rowIndex_tmp);
    int[] columnIndex = new int[num_features];
    for (int j = 0; j < num_features; j++) {
      columnIndex[j] = j;
    }

    // System.out.println(batch_index);

    // return null;
    return data.getSubMatrix(rowIndex, columnIndex);
  }
コード例 #11
0
ファイル: PearsonsCorrelation.java プロジェクト: mfwzan/astor
 /**
  * Throws MathIllegalArgumentException if the matrix does not have at least two columns and two
  * rows.
  *
  * @param matrix matrix to check for sufficiency
  * @throws MathIllegalArgumentException if there is insufficient data
  */
 private void checkSufficientData(final RealMatrix matrix) {
   int nRows = matrix.getRowDimension();
   int nCols = matrix.getColumnDimension();
   if (nRows < 2 || nCols < 2) {
     throw new MathIllegalArgumentException(
         LocalizedFormats.INSUFFICIENT_ROWS_AND_COLUMNS, nRows, nCols);
   }
 }
 public double sumMatrix(RealMatrix matrix) {
   double sum = 0.0;
   for (int i = 0; i < matrix.getRowDimension(); i++) {
     for (int j = 0; j < matrix.getColumnDimension(); j++) {
       sum += matrix.getEntry(i, j);
     }
   }
   return sum;
 }
 public double[][] residuals() {
   double[][] resid = new double[nItems][nItems];
   for (int i = 0; i < SIGMA.getRowDimension(); i++) {
     for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
       resid[i][j] = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
     }
   }
   return resid;
 }
コード例 #14
0
    public double valueAt(double[] param) {
      double[] sdInv = new double[nVariables];

      for (int i = 0; i < nVariables; i++) {
        R.setEntry(i, i, 1.0 - param[i]);
        sdInv[i] = 1.0 / Sinv.getEntry(i, i);
      }

      DiagonalMatrix diagSdInv = new DiagonalMatrix(sdInv);

      EigenDecomposition eigen = new EigenDecomposition(R);
      RealMatrix eigenVectors = eigen.getV().getSubMatrix(0, nVariables - 1, 0, nFactors - 1);

      double[] ev = new double[nFactors];
      for (int i = 0; i < nFactors; i++) {
        ev[i] = Math.sqrt(eigen.getRealEigenvalue(i));
      }
      DiagonalMatrix evMatrix =
          new DiagonalMatrix(
              ev); // USE Apache version of Diagonal matrix when upgrade to version 3.2
      RealMatrix LAMBDA = eigenVectors.multiply(evMatrix);
      RealMatrix SIGMA = (LAMBDA.multiply(LAMBDA.transpose()));

      double value = 0.0;
      RealMatrix DIF = R.subtract(SIGMA);
      for (int i = 0; i < DIF.getRowDimension(); i++) {
        for (int j = 0; j < DIF.getColumnDimension(); j++) {
          value = DIF.getEntry(i, j);
          DIF.setEntry(i, j, Math.pow(value, 2));
        }
      }

      RealMatrix RESID = diagSdInv.multiply(DIF).multiply(diagSdInv);

      double sum = 0.0;
      for (int i = 0; i < RESID.getRowDimension(); i++) {
        for (int j = 0; j < RESID.getColumnDimension(); j++) {
          sum += RESID.getEntry(i, j);
        }
      }
      return sum;
    }
 public double[][] squaredResiduals() {
   double[][] resid = new double[nItems][nItems];
   double temp = 0.0;
   for (int i = 0; i < SIGMA.getRowDimension(); i++) {
     for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
       temp = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
       resid[i][j] = temp * temp;
     }
   }
   return resid;
 }
 public double sumSquaredElements(RealMatrix matrix) {
   double sum = 0.0;
   double v = 0.0;
   for (int i = 0; i < matrix.getRowDimension(); i++) {
     for (int j = 0; j < matrix.getColumnDimension(); j++) {
       v = matrix.getEntry(i, j);
       sum += (v * v);
     }
   }
   return sum;
 }
 public double meanSquaredResidual() {
   double ni = Double.valueOf(nItems).doubleValue();
   double temp = 0.0, sum = 0.0;
   for (int i = 0; i < SIGMA.getRowDimension(); i++) {
     for (int j = 0; j < SIGMA.getColumnDimension(); j++) {
       temp = varcov.getEntry(i, j) - SIGMA.getEntry(i, j);
       sum += temp * temp;
     }
   }
   return sum / (ni * ni);
 }
コード例 #18
0
 private static double[] calculateColumnInverseMeans(RealMatrix matrix) {
   return IntStream.range(0, matrix.getColumnDimension())
       .mapToDouble(
           i ->
               1.0
                   / IntStream.range(0, matrix.getRowDimension())
                       .mapToDouble(j -> matrix.getEntry(j, i))
                       .average()
                       .orElseThrow(
                           () ->
                               new IllegalArgumentException(
                                   "cannot calculate a average for column " + i)))
       .toArray();
 }
コード例 #19
0
 private static double[][] log2ColumnMeanCenteredCounts(
     RealMatrix matrix, double[] columnInverseMeans) {
   final double[][] result = new double[matrix.getRowDimension()][columnInverseMeans.length];
   // First we divide by column mean then becoming a ratio;
   // We also log_2 transform it at this point.
   for (int i = 0; i < result.length; i++) {
     final double[] inputRow = matrix.getRow(i);
     for (int j = 0; j < columnInverseMeans.length; j++) {
       if ((result[i][j] = inputRow[j] * columnInverseMeans[j]) < EPSILON) {
         result[i][j] = EPSILON;
       }
       result[i][j] = Math.log(result[i][j]) * INV_LN2;
     }
   }
   return result;
 }
コード例 #20
0
ファイル: PearsonsCorrelation.java プロジェクト: mfwzan/astor
 /**
  * Create a PearsonsCorrelation from a RealMatrix whose columns represent variables to be
  * correlated.
  *
  * <p>Throws MathIllegalArgumentException if the matrix does not have at least two columns and two
  * rows. Pairwise correlations are set to NaN if one of the correlates has zero variance.
  *
  * @param matrix matrix with columns representing variables to correlate
  * @throws MathIllegalArgumentException if the matrix does not contain sufficient data
  * @see #correlation(double[], double[])
  */
 public PearsonsCorrelation(RealMatrix matrix) {
   nObs = matrix.getRowDimension();
   correlationMatrix = computeCorrelationMatrix(matrix);
 }
コード例 #21
0
  @Override
  public List<MLCallbackResult> detect(
      final String user,
      final String algorithm,
      UserActivityAggModel userActivity,
      UserProfileEigenModel aModel) {
    RealMatrix inputData = userActivity.matrix();
    LOG.warn(
        "EigenBasedAnomalyDetection predictAnomaly called with dimension: "
            + inputData.getRowDimension()
            + "x"
            + inputData.getColumnDimension());

    if (aModel == null) {
      LOG.warn(
          "nothing to do as the input model does not have required values, returning from evaluating this algorithm..");
      return null;
    }

    List<MLCallbackResult> mlCallbackResults = new ArrayList<MLCallbackResult>();
    RealMatrix normalizedMat = normalizeData(inputData, aModel);

    UserCommandStatistics[] listStats = aModel.statistics();
    int colWithHighVariant = 0;

    for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
      if (listStats[j].isLowVariant() == false) {
        colWithHighVariant++;
      }
    }

    final Map<String, String> context =
        new HashMap<String, String>() {
          {
            put(UserProfileConstants.USER_TAG, user);
            put(UserProfileConstants.ALGORITHM_TAG, algorithm);
          }
        };

    Map<Integer, String> lineNoWithVariantBasedAnomalyDetection = new HashMap<Integer, String>();
    for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
      MLCallbackResult aResult = new MLCallbackResult();
      aResult.setContext(context);

      for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
        // LOG.info("mean for j=" + j + " is:" + listStats[j].getMean());
        // LOG.info("stddev for j=" + j + " is:" + listStats[j].getStddev());
        if (listStats[j].isLowVariant() == true) {
          // LOG.info(listOfCmds[j] + " is low variant");
          if (normalizedMat.getEntry(i, j) > listStats[j].getMean()) {
            lineNoWithVariantBasedAnomalyDetection.put(i, "lowVariantAnomaly");
            aResult.setAnomaly(true);
            aResult.setTimestamp(userActivity.timestamp());
            aResult.setFeature(listStats[j].getCommandName());
            aResult.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
            List<String> datapoints = new ArrayList<String>();
            double[] rowVals = inputData.getRow(i);
            for (double rowVal : rowVals) datapoints.add(rowVal + "");
            aResult.setDatapoints(datapoints);
            aResult.setId(user);
            mlCallbackResults.add(aResult);
          } else {
            aResult.setAnomaly(false);
            aResult.setTimestamp(userActivity.timestamp());
            mlCallbackResults.add(aResult);
          }
        }
      }
      // return results;
    }

    // LOG.info("results size here: " + results.length);

    // LOG.info("col with high variant: " + colWithHighVariant);
    RealMatrix finalMatWithoutLowVariantFeatures =
        new Array2DRowRealMatrix(normalizedMat.getRowDimension(), colWithHighVariant);
    // LOG.info("size of final test data: " + finalMatWithoutLowVariantFeatures.getRowDimension()
    // +"x"+ finalMatWithoutLowVariantFeatures.getColumnDimension());
    int finalMatrixRow = 0;
    int finalMatrixCol = 0;
    for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
      for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
        if (listStats[j].isLowVariant() == false) {
          finalMatWithoutLowVariantFeatures.setEntry(
              finalMatrixRow, finalMatrixCol, normalizedMat.getEntry(i, j));
          finalMatrixCol++;
        }
      }
      finalMatrixCol = 0;
      finalMatrixRow++;
    }
    RealVector[] pcs = aModel.principalComponents();
    // LOG.info("pc size: " + pcs.getRowDimension() +"x" + pcs.getColumnDimension());

    RealMatrix finalInputMatTranspose = finalMatWithoutLowVariantFeatures.transpose();

    for (int i = 0; i < finalMatWithoutLowVariantFeatures.getRowDimension(); i++) {
      if (lineNoWithVariantBasedAnomalyDetection.get(i) == null) {
        MLCallbackResult result = new MLCallbackResult();
        result.setContext(context);
        for (int sz = 0; sz < pcs.length; sz++) {
          double[] pc1 = pcs[sz].toArray();
          RealMatrix pc1Mat = new Array2DRowRealMatrix(pc1);
          RealMatrix transposePC1Mat = pc1Mat.transpose();
          RealMatrix testData =
              pc1Mat.multiply(transposePC1Mat).multiply(finalInputMatTranspose.getColumnMatrix(i));
          // LOG.info("testData size: " + testData.getRowDimension() + "x" +
          // testData.getColumnDimension());
          RealMatrix testDataTranspose = testData.transpose();
          // LOG.info("testData transpose size: " + testDataTranspose.getRowDimension() + "x" +
          // testDataTranspose.getColumnDimension());
          RealVector iRowVector = testDataTranspose.getRowVector(0);
          // RealVector pc1Vector = transposePC1Mat.getRowVector(sz);
          RealVector pc1Vector = transposePC1Mat.getRowVector(0);
          double distanceiRowAndPC1 = iRowVector.getDistance(pc1Vector);
          // LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " +
          // model.getMaxL2Norm().getEntry(sz));
          // LOG.info("model.getMaxL2Norm().getEntry(sz):" + model.getMaxL2Norm().getEntry(sz));
          if (distanceiRowAndPC1 > aModel.maximumL2Norm().getEntry(sz)) {
            // LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " +
            // model.getMaxL2Norm().getEntry(sz));
            result.setAnomaly(true);
            result.setFeature(aModel.statistics()[sz].getCommandName());
            result.setTimestamp(System.currentTimeMillis());
            result.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
            List<String> datapoints = new ArrayList<String>();
            double[] rowVals = inputData.getRow(i);
            for (double rowVal : rowVals) datapoints.add(rowVal + "");
            result.setDatapoints(datapoints);
            result.setId(user);
          }
        }
        mlCallbackResults.add(result);
      }
    }
    return mlCallbackResults;
  }
コード例 #22
0
  /**
   * Tangent normalize a coverage profile.
   *
   * <p>Notes about the Spark tangent normalization can be found in docs/PoN/
   *
   * @param pon Not {@code null}
   * @param targetFactorNormalizedCounts ReadCountCollection of counts that have already been
   *     normalized fully (typically, including the target factor normalization). I.e. a coverage
   *     profile The column names should be intact. Not {@code null} See {@link
   *     TangentNormalizer::createCoverageProfile}
   * @return never {@code null}
   */
  private static TangentNormalizationResult tangentNormalize(
      final PoN pon, final ReadCountCollection targetFactorNormalizedCounts, JavaSparkContext ctx) {

    Utils.nonNull(pon, "PoN cannot be null.");
    Utils.nonNull(targetFactorNormalizedCounts, "targetFactorNormalizedCounts cannot be null.");
    Utils.nonNull(
        targetFactorNormalizedCounts.columnNames(),
        "targetFactorNormalizedCounts column names cannot be null.");
    ParamUtils.isPositive(
        targetFactorNormalizedCounts.columnNames().size(),
        "targetFactorNormalizedCounts column names cannot be an empty list.");

    final Case2PoNTargetMapper targetMapper =
        new Case2PoNTargetMapper(targetFactorNormalizedCounts.targets(), pon.getPanelTargetNames());

    // The input counts with rows (targets) sorted so that they match the PoN's order.
    final RealMatrix tangentNormalizationRawInputCounts =
        targetMapper.fromCaseToPoNCounts(targetFactorNormalizedCounts.counts());

    // We prepare the counts for tangent normalization.
    final RealMatrix tangentNormalizationInputCounts =
        composeTangentNormalizationInputMatrix(tangentNormalizationRawInputCounts);

    if (ctx == null) {

      // Calculate the beta-hats for the input read count columns (samples).
      logger.info("Calculating beta hats...");
      final RealMatrix tangentBetaHats =
          pon.betaHats(tangentNormalizationInputCounts, true, EPSILON);

      // Actual tangent normalization step.
      logger.info(
          "Performing actual tangent normalization ("
              + tangentNormalizationInputCounts.getColumnDimension()
              + " columns)...");
      final RealMatrix tangentNormalizedCounts =
          pon.tangentNormalization(tangentNormalizationInputCounts, tangentBetaHats, true);

      // Output the tangent normalized counts.
      logger.info("Post-processing tangent normalization results...");
      final ReadCountCollection tangentNormalized =
          targetMapper.fromPoNtoCaseCountCollection(
              tangentNormalizedCounts, targetFactorNormalizedCounts.columnNames());
      final ReadCountCollection preTangentNormalized =
          targetMapper.fromPoNtoCaseCountCollection(
              tangentNormalizationInputCounts, targetFactorNormalizedCounts.columnNames());

      return new TangentNormalizationResult(
          tangentNormalized, preTangentNormalized, tangentBetaHats, targetFactorNormalizedCounts);

    } else {

      /*
      Using Spark:  the code here is a little more complex for optimization purposes.

      Please see notes in docs/PoN ...

      Ahat^T = (C^T P^T) A^T
      Therefore, C^T is the RowMatrix

      pinv: P
      panel: A
      projection: Ahat
      cases: C
      betahat: C^T P^T
      tangentNormalizedCounts: C - Ahat
       */
      final RealMatrix pinv = pon.getReducedPanelPInverseCounts();
      final RealMatrix panel = pon.getReducedPanelCounts();

      // Make the C^T a distributed matrix (RowMatrix)
      final RowMatrix caseTDistMat =
          SparkConverter.convertRealMatrixToSparkRowMatrix(
              ctx, tangentNormalizationInputCounts.transpose(), TN_NUM_SLICES_SPARK);

      // Spark local matrices (transposed)
      final Matrix pinvTLocalMat =
          new DenseMatrix(
                  pinv.getRowDimension(),
                  pinv.getColumnDimension(),
                  Doubles.concat(pinv.getData()),
                  true)
              .transpose();
      final Matrix panelTLocalMat =
          new DenseMatrix(
                  panel.getRowDimension(),
                  panel.getColumnDimension(),
                  Doubles.concat(panel.getData()),
                  true)
              .transpose();

      // Calculate the projection transpose in a distributed matrix, then convert to Apache Commons
      // matrix (not transposed)
      final RowMatrix betahatDistMat = caseTDistMat.multiply(pinvTLocalMat);
      final RowMatrix projectionTDistMat = betahatDistMat.multiply(panelTLocalMat);
      final RealMatrix projection =
          SparkConverter.convertSparkRowMatrixToRealMatrix(
                  projectionTDistMat, tangentNormalizationInputCounts.transpose().getRowDimension())
              .transpose();

      // Subtract the cases from the projection
      final RealMatrix tangentNormalizedCounts =
          tangentNormalizationInputCounts.subtract(projection);

      // Construct the result object and return it with the correct targets.
      final ReadCountCollection tangentNormalized =
          targetMapper.fromPoNtoCaseCountCollection(
              tangentNormalizedCounts, targetFactorNormalizedCounts.columnNames());
      final ReadCountCollection preTangentNormalized =
          targetMapper.fromPoNtoCaseCountCollection(
              tangentNormalizationInputCounts, targetFactorNormalizedCounts.columnNames());
      final RealMatrix tangentBetaHats =
          SparkConverter.convertSparkRowMatrixToRealMatrix(
              betahatDistMat, tangentNormalizedCounts.getColumnDimension());
      return new TangentNormalizationResult(
          tangentNormalized,
          preTangentNormalized,
          tangentBetaHats.transpose(),
          targetFactorNormalizedCounts);
    }
  }
コード例 #23
0
ファイル: Covariance.java プロジェクト: mfwzan/astor
 /**
  * Create a covariance matrix from a matrix whose columns represent covariates.
  *
  * <p>The <code>biasCorrected</code> parameter determines whether or not covariance estimates are
  * bias-corrected.
  *
  * <p>The matrix must have at least one column and two rows
  *
  * @param matrix matrix with columns representing covariates
  * @param biasCorrected true means covariances are bias-corrected
  * @throws MathIllegalArgumentException if the input matrix does not have at least two rows and
  *     one column
  */
 public Covariance(RealMatrix matrix, boolean biasCorrected) throws MathIllegalArgumentException {
   checkSufficientData(matrix);
   n = matrix.getRowDimension();
   covarianceMatrix = computeCovarianceMatrix(matrix, biasCorrected);
 }
コード例 #24
0
ファイル: ICA.java プロジェクト: prashantprakash/jclustering
  @Override
  public void process() {

    // Get image dimensions, store ICA dimensions (number of frames).
    int[] dim = ip.getDimensions();
    dimensions = dim[4];

    IJ.showStatus("ICA: reformatting data matrix...");

    // Compute number of real voxels to be used
    int n = 0;
    for (@SuppressWarnings("unused") Voxel v : ip) {
      n++;
    }

    // Create new array and fill it with the image data
    double[][] image_data = new double[n][dimensions];

    int i = 0;
    for (Voxel v : ip) {
      image_data[i++] = v.tac;
    }

    // Transpose the data matrix
    Array2DRowRealMatrix temp = new Array2DRowRealMatrix(image_data, false);
    image_data = temp.transpose().getData();
    temp = null;
    System.gc();

    // Perform the ICA computation
    IJ.showStatus("ICA: performing source separation " + "(may take some time)...");
    FastICA fi = null;

    try {
      fi = new FastICA(image_data, ican);
    } catch (FastICAException e) {
      System.out.println(e.getLocalizedMessage());
    }

    IJ.showStatus("ICA computed, reformatting results...");

    // Get projections on each dimension
    double[][] vectors = fi.getICVectors();

    // Get independent signals (to be offered as additional information)
    // It is transposed to print the columns.
    double[][] sep = fi.getSeparatingMatrix();
    RealMatrix sources = new Array2DRowRealMatrix(sep);
    sources = sources.transpose();

    Array2DRowRealMatrix result = new Array2DRowRealMatrix(vectors, false);

    // If the ICA image is to be shown, create a new image with
    // result.getRowDimension() frames and the original number of
    // x, y, z dimensions
    if (showICA) {
      ImagePlus ICA_image = RealMatrix2IJ(result, dim, this.ip, "ICA image");
      ICA_image.show();
    }

    // Please note: this is somehow incorrect. As the clustering model
    // that we are following needs one voxel -> one cluster, this step
    // below assigns each voxel to the principal component with the
    // maximum weight for its particular kinetics. In "real life", the
    // resulting images would contain the contribution of that component
    // in all voxels, but for segmentation purposes this approach is
    // chosen.
    int column_index = 0;
    for (Voxel v : ip) {

      double[] projection = result.getColumn(column_index++);

      // Every Voxel belongs to the maximum index of its projected TAC
      int max = getMaxIndex(projection) + 1;
      addTACtoCluster(v, max);
    }

    // Fill in the additionalInfo array.
    additionalInfo = new String[2];
    additionalInfo[0] = "ica_sources";
    StringBuilder sb = new StringBuilder();
    int rows = sources.getRowDimension();
    for (int j = 0; j < rows; j++) {
      double[] row = sources.getRow(j);
      sb.append(Arrays.toString(row));
      sb.append("\n");
    }
    // Remove brackets
    String tempstr = sb.toString();
    tempstr = tempstr.replace("[", "");
    tempstr = tempstr.replace("]", "");
    additionalInfo[1] = tempstr;
  }
コード例 #25
0
ファイル: CostFunction.java プロジェクト: cmxnono/Face3Dv3
  /////////////////////////////////// Constructor ////////////////////////////////////////////////
  public CostFunction(
      List<Pixel> input,
      List<Pixel> average,
      List<Pixel> model,
      RealMatrix inputFeatPts,
      RealMatrix averageFeatPts,
      RealMatrix modelFeatPts,
      float[][] eigenVectorsS,
      float[][] eigenVectorsT,
      Bitmap bmpModel,
      float[] inputAlpha,
      float[] inputBeta,
      float sigmaI,
      float sigmaF) {

    // this.featPtsIndex = readBin83PtIndex(CONFIG_DIRECTORY, INDEX83PT_FILE);
    this.input = input;
    this.average = average;
    this.model = model;
    this.inputFeatPts = inputFeatPts;
    this.averageFeatPts = averageFeatPts;
    this.modelFeatPts = modelFeatPts;
    this.k = inputFeatPts.getRowDimension(); // should be equal to 83
    this.num_points = input.size(); // should be equal to 8489

    this.eigValS = readBinFloat(SHAPE_DIRECTORY, EIG_SHAPE_FILE, 60);
    this.eigValT = readBinFloat(TEXTURE_DIRECTORY, EIG_TEXTURE_FILE, 100);
    /*this.subFSV = readBinSFSV(CONFIG_DIRECTORY, SFSV_FILE);*/
    this.landmarks83Index =
        readBin83PtIndex(
            CONFIG_DIRECTORY, INDEX83PT_FILE); // we access directly in featureShape file
    // instead of using subFSV file

    // Checking arguments
    if (num_points != 8489) {
      throw new IllegalArgumentException("num_points not equal 8489");
    }
    if (k != 83) {
      throw new IllegalArgumentException("k not equal 83");
    }
    if (input.isEmpty() || average.isEmpty() || model.isEmpty()) {
      throw new IllegalArgumentException("input or average or model list are empty");
    }
    if (input.size() != model.size()) {
      throw new IllegalArgumentException("input and model list do not have the same size");
    }
    if (input.size() != average.size()) {
      throw new IllegalArgumentException("input and average list do not have the same size");
    }
    if (average.size() != model.size()) {
      throw new IllegalArgumentException("average and model list do not have the same size");
    }
    if (averageFeatPts.getRowDimension() != k
        || averageFeatPts.getColumnDimension() != inputFeatPts.getColumnDimension()) {
      throw new IllegalArgumentException(
          "inputFeatPts and averageFeatPts do not have the same size");
    }
    if (modelFeatPts.getRowDimension() != k
        || modelFeatPts.getColumnDimension() != inputFeatPts.getColumnDimension()) {
      throw new IllegalArgumentException("inputFeatPts and modelFeatPts do not have the same size");
    }

    // Initialy populate list with 0, the value doesn't matter
    for (int h = 0; h < 500; h++) {
      this.randomList.add(0);
    }

    this.s = eigenVectorsS;
    this.t = eigenVectorsT;

    this.inputAlpha = inputAlpha;
    this.inputBeta = inputBeta;

    this.sigmaI = sigmaI;
    this.sigmaF = sigmaF;

    this.Iinput = computeIinput(); // is always the same
    this.Imodel = computeImodel();

    this.IinputBeta = computeIinputBeta(); // is always the same
    this.ImodelBeta = computeImodelBeta();

    this.dxModel = computeSobelGx(bmpModel);
    this.dyModel = computeSobelGy(bmpModel);
    // saveBitmaptoPNG(TEXTURE_DIRECTORY, "modelFace2DGx.png", bmpModelGx); //save
    // saveBitmaptoPNG(TEXTURE_DIRECTORY, "modelFace2DGy.png", bmpModelGy); //save

    this.E = computeE();

    this.alpha = inputAlpha; // Initialize
    this.beta = inputBeta; // Initialize
    computeAlpha(); // Compute 60 alpha values output
    computeBeta(); // Compute 100 beta values output

    /*        Collections.sort(randomList);
    for(int idx : randomList) {
        Log.d(TAG,"randomList = " + idx);
    }*/
  }