public Object call() throws MatrixException {
   Matrix m =
       getMatrixObject()
           .getMatrix()
           .bootstrap(getNewOrLink(), GUIUtil.getInt("Number of samples", 1, 10000000));
   m.showGUI();
   return m;
 }
 public final Matrix copy() throws MatrixException {
   Object[] result = new Object[values.length];
   System.arraycopy(values, 0, result, 0, values.length);
   Matrix m = new DefaultDenseObjectMatrix2D(result, rows, cols);
   if (getAnnotation() != null) {
     m.setAnnotation(getAnnotation().clone());
   }
   return m;
 }
 public Long call() throws Exception {
   Matrix newColumn = replaceInColumn(getSource(), bestGuess, column);
   synchronized (imputed) {
     for (int r = 0; r < newColumn.getRowCount(); r++) {
       imputed.setAsDouble(newColumn.getAsDouble(r, 0), r, column);
     }
   }
   return column;
 }
Example #4
0
  @Override
  public Map<Object, Object> calculateObjects(Map<Object, Object> input) throws MatrixException {
    Map<Object, Object> result = new HashMap<Object, Object>();

    Matrix source1 = MathUtil.getMatrix(input.get(SOURCE1));
    Matrix source2 = MathUtil.getMatrix(input.get(SOURCE2));

    result.put(TARGET, source1.mtimes(source2));

    return result;
  }
 public Xor(Matrix m1, Matrix m2) {
   super(m1, m2);
   if (m2.isScalar() && !Coordinates.equals(m1.getSize(), m2.getSize())) {
     getSources()[1] = MatrixFactory.fill(m2.getAsBoolean(0, 0), m1.getSize());
   } else if (m1.isScalar() && !Coordinates.equals(m1.getSize(), m2.getSize())) {
     getSources()[0] = MatrixFactory.fill(m1.getAsBoolean(0, 0), m2.getSize());
   }
 }
 public DefaultDenseObjectMatrix2D(Matrix m) throws MatrixException {
   this.rows = (int) m.getRowCount();
   this.cols = (int) m.getColumnCount();
   this.size = new long[] {rows, cols};
   if (m instanceof DefaultDenseObjectMatrix2D) {
     Object[] v = ((DefaultDenseObjectMatrix2D) m).values;
     this.values = new Object[v.length];
     System.arraycopy(v, 0, this.values, 0, v.length);
   } else {
     this.values = new Object[rows * cols];
     for (long[] c : m.allCoordinates()) {
       setObject(m.getAsObject(c), c);
     }
   }
 }
  public Object call() {
    try {
      String s =
          JOptionPane.showInputDialog(
              getComponent(),
              "Enter the rows and columns to select, e.g. 1,3-5;4-5,7",
              "Select",
              JOptionPane.QUESTION_MESSAGE);

      Matrix m = getMatrixObject().getMatrix().select(getNewOrLink(), s);
      m.showGUI();
      return m;
    } catch (Exception e) {
      e.printStackTrace();
      return null;
    }
  }
  public Map<String, Object> calculateObjects(Map<String, Object> input) {
    Map<String, Object> result = new HashMap<String, Object>();

    Matrix source = MathUtil.getMatrix(input.get(SOURCE));

    Matrix target = Matrix.Factory.zeros(source.getRowCount(), 1);

    long cols = source.getColumnCount();
    long rows = source.getRowCount();
    for (int k = 0; k < rows; k++) {
      int tp = 0;
      int fn = 0;
      for (int r = 0; r < rows; r++) {
        for (int c = 0; c < cols; c++) {
          int count = source.getAsInt(r, c);
          boolean expected = r == k;
          boolean predicted = c == k;
          if (expected && predicted) {
            tp += count;
          } else if (expected && (!predicted)) {
            fn += count;
          }
        }
      }
      target.setAsDouble(MathUtil.sensitivity(tp, fn), k, 0);
    }
    result.put(TARGET, target);
    return result;
  }
 public double getDouble(long... coordinates) {
   if (inv == null) {
     DenseMatrix A = new MTJDenseDoubleMatrix2D(getSource()).getWrappedObject();
     DenseMatrix I = Matrices.identity((int) getSource().getColumnCount());
     DenseMatrix AI = I.copy();
     inv = new MTJDenseDoubleMatrix2D((DenseMatrix) A.solve(I, AI));
   }
   return inv.getAsDouble(coordinates);
 }
 public double getDouble(long... coordinates) {
   if (mean == null) {
     mean = new Mean(getDimension(), true, getSource()).calcNew();
   }
   double v = getSource().getAsDouble(coordinates);
   if (MathUtil.isNaNOrInfinite(v)) {
     switch (getDimension()) {
       case ALL:
         return mean.getAsDouble(0, 0);
       case ROW:
         return mean.getAsDouble(0, coordinates[COLUMN]);
       case COLUMN:
         return mean.getAsDouble(coordinates[ROW], 0);
     }
   } else {
     return v;
   }
   return 0.0;
 }
 public double getDouble(long... coordinates) {
   if (imputed == null) {
     createMatrix();
   }
   double v = getSource().getAsDouble(coordinates);
   if (MathUtil.isNaNOrInfinite(v)) {
     return imputed.getAsDouble(coordinates);
   } else {
     return v;
   }
 }
  public Map<String, Object> calculateObjects(Map<String, Object> input) {
    int dimension = defaultDimension;
    boolean ignoreNaN = defaultIgnoreNaN;

    Map<String, Object> result = new HashMap<String, Object>();

    Matrix source = MathUtil.getMatrix(input.get(SOURCE));
    Object o2 = input.get(DIMENSION);
    if (o2 != null) {
      dimension = MathUtil.getInt(o2);
    }
    Object o3 = input.get(IGNORENAN);
    if (o3 != null) {
      ignoreNaN = MathUtil.getBoolean(o3);
    }

    Matrix target = source.mean(Ret.NEW, dimension, ignoreNaN);
    result.put(TARGET, target);
    return result;
  }
  private Matrix calculate() {
    wordMapping = new DefaultMapMatrix<String, Long>();
    Matrix m = getSource();
    for (long[] c : m.availableCoordinates()) {
      String s = m.getAsString(c);
      if (s != null) {
        String[] words = s.split("\\s+");
        for (String w : words) {
          if (w.length() == 0) {
            continue;
          }
          Long i = wordMapping.get(w);
          if (i == null) {
            wordMapping.put(w, wordMapping.getRowCount());
          }
        }
      }
    }
    result = new DefaultSparseLongMatrix(m.getRowCount(), wordMapping.getRowCount());

    long rowCount = m.getRowCount();
    long colCount = m.getColumnCount();
    for (long row = 0; row < rowCount; row++) {
      for (long col = 0; col < colCount; col++) {
        String string = m.getAsString(row, col);
        if (string != null && string.length() > 0) {
          String[] words = string.split("[\\s]+");
          for (String w : words) {
            if (w.length() == 0) {
              continue;
            }
            long i = wordMapping.get(w);
            int count = result.getAsInt(row, i);
            result.setAsInt(++count, row, i);
          }
        }
      }
    }
    return result;
  }
  public void asLatex() throws IOException {
    final String EOL = System.getProperty("line.separator");
    final Writer writer = getWriter();
    final Matrix matrix = getMatrix();

    final long rowCount = matrix.getRowCount();
    final long colCount = matrix.getColumnCount();

    writer.write("\\begin{table}[!ht]" + EOL);
    writer.write("\\centering" + EOL);

    if (matrix.getLabelObject() != null) {
      writer.write(
          "\\caption{"
              + UJMPFormat.getSingleLineInstance().format(matrix.getLabelObject())
              + "}"
              + EOL);
    }

    StringBuilder buf = new StringBuilder();
    for (long i = matrix.getColumnCount() - 1; i != -1; i--) {
      buf.append('c');
    }
    String alignment = buf.toString();

    writer.write("\\begin{tabular}{" + alignment + "}" + EOL);
    writer.write("\\toprule" + EOL);

    for (int row = 0; row < rowCount; row++) {
      for (int col = 0; col < colCount; col++) {
        writer.write(UJMPFormat.getSingleLineInstance().format(matrix.getAsObject(row, col)));

        if (col < colCount - 1) {
          writer.write(" & ");
        }
      }
      writer.write(" \\\\" + EOL);
    }

    writer.write("\\bottomrule" + EOL);
    writer.write("\\end{tabular}" + EOL);
    writer.write("\\end{table}" + EOL);
  }
  private static Matrix replaceInColumn(Matrix original, Matrix firstGuess, long column) {

    Matrix x = firstGuess.deleteColumns(Ret.NEW, column);
    Matrix y = original.selectColumns(Ret.NEW, column);

    List<Long> missingRows = new ArrayList<Long>();
    for (long i = y.getRowCount(); --i >= 0; ) {
      double v = y.getAsDouble(i, 0);
      if (MathUtil.isNaNOrInfinite(v)) {
        missingRows.add(i);
      }
    }

    if (missingRows.isEmpty()) {
      return y;
    }

    Matrix xdel = x.deleteRows(Ret.NEW, missingRows);
    DenseDoubleMatrix2D bias1 = DenseDoubleMatrix2D.Factory.ones(xdel.getRowCount(), 1);
    Matrix xtrain = Matrix.Factory.horCat(xdel, bias1);
    Matrix ytrain = y.deleteRows(Ret.NEW, missingRows);

    Matrix xinv = xtrain.pinv();
    Matrix b = xinv.mtimes(ytrain);
    DenseDoubleMatrix2D bias2 = DenseDoubleMatrix2D.Factory.ones(x.getRowCount(), 1);
    Matrix yPredicted = Matrix.Factory.horCat(x, bias2).mtimes(b);

    // set non-missing values back to original values
    for (int row = 0; row < y.getRowCount(); row++) {
      double v = y.getAsDouble(row, 0);
      if (!Double.isNaN(v)) {
        yPredicted.setAsDouble(v, row, 0);
      }
    }

    return yPredicted;
  }
 public long getLong(long... coordinates) {
   if (result == null) {
     result = calculate();
   }
   return result.getAsLong(coordinates);
 }
 public long[] getSize() {
   if (result == null) {
     result = calculate();
   }
   return result.getSize();
 }
Example #18
0
  public void calcscores() {
    Matrix mbase, mbaseavg;
    Matrix mtemp;
    double sco, div;
    @SuppressWarnings("unused")
    double scodef, divdef;
    mbase = clustinit.vtestsm;
    mbaseavg = clustinit.avgsm;
    scorevar = new double[(int) mbase.getRowCount()];
    scorestabdet = new double[(int) mbase.getRowCount()];
    scorevardef = new double[(int) mbase.getRowCount()];
    scorestabdesc = 0;
    scorestabpop = 0;
    for (int i = 0; i < 3; i++) {
      scorevar[i] = 0;
      scorevardef[i] = 0;
    }
    for (int i = 3; i < mbase.getRowCount(); i++) {

      mtemp = mbase.selectRows(Ret.LINK, i);
      sco = mtemp.abs(Ret.LINK).getMeanValue();
      sco = sco * mbaseavg.selectRows(Ret.LINK, i).getStdValue();
      div = mbaseavg.abs(Ret.LINK).selectRows(Ret.LINK, i).getMeanValue();
      if (div > 0) {
        sco = sco / div;
      } else {
        sco = 0;
      }
      scorevar[i] = sco;
    }
    mbase = clustinit.vtestsmdef;
    mbaseavg = clustinit.avgsmdef;
    for (int i = 3; i < mbase.getRowCount(); i++) {
      mtemp = mbase.selectRows(Ret.LINK, i);
      sco = mtemp.abs(Ret.LINK).getMeanValue();
      sco = sco * mbaseavg.selectRows(Ret.LINK, i).getStdValue();
      div = mbaseavg.abs(Ret.LINK).selectRows(Ret.LINK, i).getMeanValue();
      if (div > 0) {
        sco = sco / div;
      } else {
        sco = 0;
      }
      scorevardef[i] = sco;
    }
    scorestabdesc = 0;
    @SuppressWarnings("unused")
    int nbvarninit;
    for (int i = 3; i < mbase.getRowCount(); i++) {
      scorestabdet[i] = 0;
      for (int j = 0; j < mbase.getColumnCount(); j++) {
        scorestabdet[i] +=
            Math.abs(
                    clustinit.vtestsm.getAsDouble(i, j)
                        - clustinit.vtestsm.getAsDouble(i, clustinit.idtickinit))
                / Math.max(
                    Math.abs(clustinit.vtestsm.getAsDouble(i, j)),
                    Math.abs(clustinit.vtestsm.getAsDouble(i, clustinit.idtickinit)))
                / mbase.getColumnCount();
      }
      scorestabdet[i] = 1 - scorestabdet[i];
      if (!Double.isNaN(scorestabdet[i])) {
        scorestabdesc = scorestabdesc + scorestabdet[i] / (mbase.getRowCount() - 3);
      }
    }
    scorestabpop = 0;
    Cluster nc;
    for (int j = 0; j < clustHistDef.size(); j++) {
      nc = clustHistDef.get(j);
      scorestabpop +=
          (double) clustinit.getNumberOfCommonComponents(nc)
              / (double) Math.max(clustinit.getComponentIds().size(), nc.getComponentIds().size());
    }
    scorestabpop = (double) scorestabpop / clustinit.vtestsm.getColumnCount();

    long idColumn = mbase.getRowForLabel(Cluster.ID_C_NAME);
    long classLabelColumn = mbase.getRowForLabel(Cluster.CLASS_LABEL_C_NAME);
    long labelColorColumn = mbase.getRowForLabel("LABEL-COLOR");
    for (int i = 3; i < mbase.getRowCount(); i++) {
      Pattern p = Pattern.compile("CLASS_LABEL");
      Matcher m = p.matcher(mbase.getRowLabel(i));
      if ((i == idColumn || i == classLabelColumn || i == labelColorColumn || m.lookingAt())) {
        scorevardef[i] = 0;
        scorevar[i] = 0;
      }
    }
  }
  private void createMatrix() {
    try {
      ExecutorService executor =
          Executors.newFixedThreadPool(UJMPSettings.getInstance().getNumberOfThreads());

      Matrix x = getSource();

      double valueCount = x.getValueCount();
      long missingCount = (long) x.countMissing(Ret.NEW, Matrix.ALL).getEuklideanValue();
      double percent = ((int) Math.round((missingCount * 1000.0 / valueCount))) / 10.0;
      System.out.println("missing values: " + missingCount + " (" + percent + "%)");
      System.out.println("============================================");

      if (bestGuess == null) {
        bestGuess = getSource().impute(Ret.NEW, ImputationMethod.RowMean);
      }

      int run = 0;
      double d;
      do {
        System.out.println("Iteration " + run++);

        List<Future<Long>> futures = new ArrayList<Future<Long>>();

        imputed = Matrix.Factory.zeros(x.getSize());

        long t0 = System.currentTimeMillis();

        for (long c = 0; c < x.getColumnCount(); c++) {
          if (containsMissingValues(c)) {
            futures.add(executor.submit(new PredictColumn(c)));
          }
        }

        for (Future<Long> f : futures) {
          Long completedCols = f.get();
          long elapsedTime = System.currentTimeMillis() - t0;
          long remainingCols = x.getColumnCount() - completedCols;
          double colsPerMillisecond = (double) (completedCols + 1) / (double) elapsedTime;
          long remainingTime = (long) (remainingCols / colsPerMillisecond / 1000.0);
          System.out.println(
              (completedCols * 1000 / x.getColumnCount() / 10.0)
                  + "% completed ("
                  + remainingTime
                  + " seconds remaining)");
        }

        Matrix newBestGuess = bestGuess.times(decay).plus(imputed.times(1 - decay));

        for (int r = 0; r < getSource().getRowCount(); r++) {
          for (int c = 0; c < getSource().getColumnCount(); c++) {
            double value = getSource().getAsDouble(r, c);
            if (!MathUtil.isNaNOrInfinite(value)) {
              newBestGuess.setAsDouble(value, r, c);
            }
          }
        }

        d = newBestGuess.euklideanDistanceTo(bestGuess, true) / missingCount;
        System.out.println("delta: " + d);
        System.out.println("============================================");

        bestGuess = newBestGuess;
        bestGuess.exportTo().file(tempFile).asDenseCSV();

      } while (delta < d);

      executor.shutdown();

      imputed = bestGuess;

      if (imputed.containsMissingValues()) {
        throw new RuntimeException("Matrix has still missing values after imputation");
      }

    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
 public Xor(Matrix m1, boolean v2) throws MatrixException {
   this(m1, MatrixFactory.fill(v2, m1.getSize()));
 }
 public Xor(boolean v1, Matrix m2) throws MatrixException {
   this(MatrixFactory.fill(v1, m2.getSize()), m2);
 }
Example #22
0
  @Override
  protected double sampleAlpha(
      Matrix direction, Matrix Aobj, Matrix objConst, double alphaLow, double alphaHigh) {
    // Construct distribution over the randomly sampled line given by the direction and the bounds
    // [alphaLow, alphaHigh]
    Matrix AobjD = Aobj.mtimes(direction);
    Matrix AobjP = Aobj.mtimes(currentPt);

    OpenLongObjectHashMap alphaMap = new OpenLongObjectHashMap();
    long alphaLowBin = Math.round(alphaLow * alphaRoundingScheme);
    ObjFunHandle alphaLowHandle = ObjFunHandle.getHandle(alphaLowBin, alphaMap);

    for (int i = 0; i < Aobj.getRowCount(); i++) {
      double rate = AobjD.getAsDouble(i, 0);
      double constant = AobjP.getAsDouble(i, 0) + objConst.getAsDouble(i, 0);
      double alpha = (-constant) / rate;
      long alphaBin = Math.round(alpha * alphaRoundingScheme);
      log.trace("Alpha {}", alpha);
      log.trace("Rate {} Constant {}", rate, constant);
      if (rate > epsilon && alpha < alphaHigh) {
        ObjFunHandle h;
        if (alpha <= alphaLow) {
          // This evidence is active over the entire interval
          h = alphaLowHandle;
        } else {
          // This evidence will get active once we reach alpha in the interval
          h = ObjFunHandle.getHandle(alphaBin, alphaMap);
        }
        h.increaseBy(rate, constant);
      } else if (rate < -epsilon && alpha > alphaLow) {
        alphaLowHandle.increaseBy(rate, constant);
        if (alpha >= alphaHigh) {
          // This evidence is active over the entire interval
        } else {
          // This evidence will get inactive once we reach alpha in the interval
          ObjFunHandle h = ObjFunHandle.getHandle(alphaBin, alphaMap);
          h.increaseBy(-rate, -constant);
        }
      } else { // Considered 0
        alphaLowHandle.increaseBy(0.0, constant);
      }
    }
    ObjFunHandle.getHandle(Math.round(alphaHigh * alphaRoundingScheme), alphaMap);
    AobjD = null;
    AobjP = null;

    int length = alphaMap.size();
    LongArrayList alphaBinsL = new LongArrayList(length);
    ObjectArrayList handlesL = new ObjectArrayList(length);
    alphaMap.pairsSortedByKey(alphaBinsL, handlesL);
    assert alphaBinsL.size() == length && handlesL.size() == length;

    double[] logcumulative = new double[length];
    double[] cumRate = new double[length - 1];
    double[] cumConst = new double[length - 1];
    long[] alphaBins = alphaBinsL.elements();
    Object[] handles = handlesL.elements();

    log.trace("alphaBins: {}", Arrays.toString(alphaBins));
    log.trace("handles: {}", Arrays.toString(handles));

    // Now compute the cumulative probability distribution
    ObjFunHandle base = (ObjFunHandle) handles[0];
    double baserate = base.getRate();
    double baseconstant = base.getConstant();
    double basealpha = alphaBins[0] / (double) alphaRoundingScheme;
    double baselogdivisor = baserate * basealpha + baseconstant;

    logcumulative[0] = 0.0;
    double rate = 0.0;
    double constant = 0.0;
    for (int i = 1; i < length; i++) {
      ObjFunHandle h = (ObjFunHandle) handles[i - 1];
      rate += h.getRate();
      cumRate[i - 1] = rate;
      constant += h.getConstant();
      cumConst[i - 1] = constant;
      double aL = alphaBins[i - 1] / (double) alphaRoundingScheme,
          aU = alphaBins[i] / (double) alphaRoundingScheme;
      double increment;
      if (rate < -epsilon || rate > epsilon)
        increment =
            1.0
                / rate
                * (Math.exp(baselogdivisor - rate * aL - constant)
                    - Math.exp(baselogdivisor - rate * aU - constant));
      else increment = (aU - aL) * Math.exp(baselogdivisor - constant);
      assert increment > -epsilon : increment;
      logcumulative[i] = logcumulative[i - 1] + increment;
    }
    double total = logcumulative[length - 1];
    double randomPt = Math.random() * total;
    log.trace("Random Pt {}", randomPt);

    log.trace("logcumulative: {}", Arrays.toString(logcumulative));

    int findIndex = Arrays.binarySearch(logcumulative, randomPt);
    log.trace("Find Index {}", findIndex);
    double alphaPt;
    if (findIndex >= 0) alphaPt = alphaBins[findIndex] / (double) alphaRoundingScheme;
    else {
      findIndex = -findIndex - 1;
      assert findIndex > 0 : findIndex;
      double aL = alphaBins[findIndex - 1] / (double) alphaRoundingScheme;
      final double crate = cumRate[findIndex - 1];
      final double cconstant = cumConst[findIndex - 1];
      log.trace("R {}, c {} aL " + aL, crate, cconstant);
      if (crate < -epsilon || crate > epsilon) {
        double targetValue =
            -randomPt
                + logcumulative[findIndex - 1]
                + 1.0 / crate * Math.exp(-cconstant - crate * aL + baselogdivisor);
        log.trace("Target value {}", targetValue);
        log.trace("Log: {}, inside log {}", Math.log(crate * targetValue), crate * targetValue);
        alphaPt = -1.0 / crate * (Math.log(crate * targetValue) + cconstant - baselogdivisor);
      } else {
        double kcons = Math.exp(baselogdivisor - cconstant);
        alphaPt = randomPt / kcons - logcumulative[findIndex - 1] / kcons + aL;
      }
      assert !Double.isNaN(alphaPt)
          : "C: "
              + cconstant
              + " | R: "
              + crate
              + " Pt: "
              + randomPt
              + " aL: "
              + aL
              + "  C0:"
              + logcumulative[findIndex - 1];
    }
    assert alphaPt >= alphaLow - epsilon && alphaPt <= alphaHigh + epsilon : alphaPt;
    alphaPt = Math.max(alphaPt, alphaLow);
    alphaPt = Math.min(alphaPt, alphaHigh);
    log.trace("Sampled alpha {}", alphaPt);

    return alphaPt;
  }
 public Object call() throws MatrixException {
   Matrix m = getMatrixObject().getMatrix().sin(getRet());
   m.showGUI();
   return m;
 }