public Map<String, Object> calculateObjects(Map<String, Object> input) {
    Map<String, Object> result = new HashMap<String, Object>();

    Matrix source = MathUtil.getMatrix(input.get(SOURCE));

    Matrix target = Matrix.Factory.zeros(source.getRowCount(), 1);

    long cols = source.getColumnCount();
    long rows = source.getRowCount();
    for (int k = 0; k < rows; k++) {
      int tp = 0;
      int fn = 0;
      for (int r = 0; r < rows; r++) {
        for (int c = 0; c < cols; c++) {
          int count = source.getAsInt(r, c);
          boolean expected = r == k;
          boolean predicted = c == k;
          if (expected && predicted) {
            tp += count;
          } else if (expected && (!predicted)) {
            fn += count;
          }
        }
      }
      target.setAsDouble(MathUtil.sensitivity(tp, fn), k, 0);
    }
    result.put(TARGET, target);
    return result;
  }
 public Long call() throws Exception {
   Matrix newColumn = replaceInColumn(getSource(), bestGuess, column);
   synchronized (imputed) {
     for (int r = 0; r < newColumn.getRowCount(); r++) {
       imputed.setAsDouble(newColumn.getAsDouble(r, 0), r, column);
     }
   }
   return column;
 }
  private Matrix calculate() {
    wordMapping = new DefaultMapMatrix<String, Long>();
    Matrix m = getSource();
    for (long[] c : m.availableCoordinates()) {
      String s = m.getAsString(c);
      if (s != null) {
        String[] words = s.split("\\s+");
        for (String w : words) {
          if (w.length() == 0) {
            continue;
          }
          Long i = wordMapping.get(w);
          if (i == null) {
            wordMapping.put(w, wordMapping.getRowCount());
          }
        }
      }
    }
    result = new DefaultSparseLongMatrix(m.getRowCount(), wordMapping.getRowCount());

    long rowCount = m.getRowCount();
    long colCount = m.getColumnCount();
    for (long row = 0; row < rowCount; row++) {
      for (long col = 0; col < colCount; col++) {
        String string = m.getAsString(row, col);
        if (string != null && string.length() > 0) {
          String[] words = string.split("[\\s]+");
          for (String w : words) {
            if (w.length() == 0) {
              continue;
            }
            long i = wordMapping.get(w);
            int count = result.getAsInt(row, i);
            result.setAsInt(++count, row, i);
          }
        }
      }
    }
    return result;
  }
  private static Matrix replaceInColumn(Matrix original, Matrix firstGuess, long column) {

    Matrix x = firstGuess.deleteColumns(Ret.NEW, column);
    Matrix y = original.selectColumns(Ret.NEW, column);

    List<Long> missingRows = new ArrayList<Long>();
    for (long i = y.getRowCount(); --i >= 0; ) {
      double v = y.getAsDouble(i, 0);
      if (MathUtil.isNaNOrInfinite(v)) {
        missingRows.add(i);
      }
    }

    if (missingRows.isEmpty()) {
      return y;
    }

    Matrix xdel = x.deleteRows(Ret.NEW, missingRows);
    DenseDoubleMatrix2D bias1 = DenseDoubleMatrix2D.Factory.ones(xdel.getRowCount(), 1);
    Matrix xtrain = Matrix.Factory.horCat(xdel, bias1);
    Matrix ytrain = y.deleteRows(Ret.NEW, missingRows);

    Matrix xinv = xtrain.pinv();
    Matrix b = xinv.mtimes(ytrain);
    DenseDoubleMatrix2D bias2 = DenseDoubleMatrix2D.Factory.ones(x.getRowCount(), 1);
    Matrix yPredicted = Matrix.Factory.horCat(x, bias2).mtimes(b);

    // set non-missing values back to original values
    for (int row = 0; row < y.getRowCount(); row++) {
      double v = y.getAsDouble(row, 0);
      if (!Double.isNaN(v)) {
        yPredicted.setAsDouble(v, row, 0);
      }
    }

    return yPredicted;
  }
 public DefaultDenseObjectMatrix2D(Matrix m) throws MatrixException {
   this.rows = (int) m.getRowCount();
   this.cols = (int) m.getColumnCount();
   this.size = new long[] {rows, cols};
   if (m instanceof DefaultDenseObjectMatrix2D) {
     Object[] v = ((DefaultDenseObjectMatrix2D) m).values;
     this.values = new Object[v.length];
     System.arraycopy(v, 0, this.values, 0, v.length);
   } else {
     this.values = new Object[rows * cols];
     for (long[] c : m.allCoordinates()) {
       setObject(m.getAsObject(c), c);
     }
   }
 }
  public void asLatex() throws IOException {
    final String EOL = System.getProperty("line.separator");
    final Writer writer = getWriter();
    final Matrix matrix = getMatrix();

    final long rowCount = matrix.getRowCount();
    final long colCount = matrix.getColumnCount();

    writer.write("\\begin{table}[!ht]" + EOL);
    writer.write("\\centering" + EOL);

    if (matrix.getLabelObject() != null) {
      writer.write(
          "\\caption{"
              + UJMPFormat.getSingleLineInstance().format(matrix.getLabelObject())
              + "}"
              + EOL);
    }

    StringBuilder buf = new StringBuilder();
    for (long i = matrix.getColumnCount() - 1; i != -1; i--) {
      buf.append('c');
    }
    String alignment = buf.toString();

    writer.write("\\begin{tabular}{" + alignment + "}" + EOL);
    writer.write("\\toprule" + EOL);

    for (int row = 0; row < rowCount; row++) {
      for (int col = 0; col < colCount; col++) {
        writer.write(UJMPFormat.getSingleLineInstance().format(matrix.getAsObject(row, col)));

        if (col < colCount - 1) {
          writer.write(" & ");
        }
      }
      writer.write(" \\\\" + EOL);
    }

    writer.write("\\bottomrule" + EOL);
    writer.write("\\end{tabular}" + EOL);
    writer.write("\\end{table}" + EOL);
  }
Esempio n. 7
0
  @Override
  protected double sampleAlpha(
      Matrix direction, Matrix Aobj, Matrix objConst, double alphaLow, double alphaHigh) {
    // Construct distribution over the randomly sampled line given by the direction and the bounds
    // [alphaLow, alphaHigh]
    Matrix AobjD = Aobj.mtimes(direction);
    Matrix AobjP = Aobj.mtimes(currentPt);

    OpenLongObjectHashMap alphaMap = new OpenLongObjectHashMap();
    long alphaLowBin = Math.round(alphaLow * alphaRoundingScheme);
    ObjFunHandle alphaLowHandle = ObjFunHandle.getHandle(alphaLowBin, alphaMap);

    for (int i = 0; i < Aobj.getRowCount(); i++) {
      double rate = AobjD.getAsDouble(i, 0);
      double constant = AobjP.getAsDouble(i, 0) + objConst.getAsDouble(i, 0);
      double alpha = (-constant) / rate;
      long alphaBin = Math.round(alpha * alphaRoundingScheme);
      log.trace("Alpha {}", alpha);
      log.trace("Rate {} Constant {}", rate, constant);
      if (rate > epsilon && alpha < alphaHigh) {
        ObjFunHandle h;
        if (alpha <= alphaLow) {
          // This evidence is active over the entire interval
          h = alphaLowHandle;
        } else {
          // This evidence will get active once we reach alpha in the interval
          h = ObjFunHandle.getHandle(alphaBin, alphaMap);
        }
        h.increaseBy(rate, constant);
      } else if (rate < -epsilon && alpha > alphaLow) {
        alphaLowHandle.increaseBy(rate, constant);
        if (alpha >= alphaHigh) {
          // This evidence is active over the entire interval
        } else {
          // This evidence will get inactive once we reach alpha in the interval
          ObjFunHandle h = ObjFunHandle.getHandle(alphaBin, alphaMap);
          h.increaseBy(-rate, -constant);
        }
      } else { // Considered 0
        alphaLowHandle.increaseBy(0.0, constant);
      }
    }
    ObjFunHandle.getHandle(Math.round(alphaHigh * alphaRoundingScheme), alphaMap);
    AobjD = null;
    AobjP = null;

    int length = alphaMap.size();
    LongArrayList alphaBinsL = new LongArrayList(length);
    ObjectArrayList handlesL = new ObjectArrayList(length);
    alphaMap.pairsSortedByKey(alphaBinsL, handlesL);
    assert alphaBinsL.size() == length && handlesL.size() == length;

    double[] logcumulative = new double[length];
    double[] cumRate = new double[length - 1];
    double[] cumConst = new double[length - 1];
    long[] alphaBins = alphaBinsL.elements();
    Object[] handles = handlesL.elements();

    log.trace("alphaBins: {}", Arrays.toString(alphaBins));
    log.trace("handles: {}", Arrays.toString(handles));

    // Now compute the cumulative probability distribution
    ObjFunHandle base = (ObjFunHandle) handles[0];
    double baserate = base.getRate();
    double baseconstant = base.getConstant();
    double basealpha = alphaBins[0] / (double) alphaRoundingScheme;
    double baselogdivisor = baserate * basealpha + baseconstant;

    logcumulative[0] = 0.0;
    double rate = 0.0;
    double constant = 0.0;
    for (int i = 1; i < length; i++) {
      ObjFunHandle h = (ObjFunHandle) handles[i - 1];
      rate += h.getRate();
      cumRate[i - 1] = rate;
      constant += h.getConstant();
      cumConst[i - 1] = constant;
      double aL = alphaBins[i - 1] / (double) alphaRoundingScheme,
          aU = alphaBins[i] / (double) alphaRoundingScheme;
      double increment;
      if (rate < -epsilon || rate > epsilon)
        increment =
            1.0
                / rate
                * (Math.exp(baselogdivisor - rate * aL - constant)
                    - Math.exp(baselogdivisor - rate * aU - constant));
      else increment = (aU - aL) * Math.exp(baselogdivisor - constant);
      assert increment > -epsilon : increment;
      logcumulative[i] = logcumulative[i - 1] + increment;
    }
    double total = logcumulative[length - 1];
    double randomPt = Math.random() * total;
    log.trace("Random Pt {}", randomPt);

    log.trace("logcumulative: {}", Arrays.toString(logcumulative));

    int findIndex = Arrays.binarySearch(logcumulative, randomPt);
    log.trace("Find Index {}", findIndex);
    double alphaPt;
    if (findIndex >= 0) alphaPt = alphaBins[findIndex] / (double) alphaRoundingScheme;
    else {
      findIndex = -findIndex - 1;
      assert findIndex > 0 : findIndex;
      double aL = alphaBins[findIndex - 1] / (double) alphaRoundingScheme;
      final double crate = cumRate[findIndex - 1];
      final double cconstant = cumConst[findIndex - 1];
      log.trace("R {}, c {} aL " + aL, crate, cconstant);
      if (crate < -epsilon || crate > epsilon) {
        double targetValue =
            -randomPt
                + logcumulative[findIndex - 1]
                + 1.0 / crate * Math.exp(-cconstant - crate * aL + baselogdivisor);
        log.trace("Target value {}", targetValue);
        log.trace("Log: {}, inside log {}", Math.log(crate * targetValue), crate * targetValue);
        alphaPt = -1.0 / crate * (Math.log(crate * targetValue) + cconstant - baselogdivisor);
      } else {
        double kcons = Math.exp(baselogdivisor - cconstant);
        alphaPt = randomPt / kcons - logcumulative[findIndex - 1] / kcons + aL;
      }
      assert !Double.isNaN(alphaPt)
          : "C: "
              + cconstant
              + " | R: "
              + crate
              + " Pt: "
              + randomPt
              + " aL: "
              + aL
              + "  C0:"
              + logcumulative[findIndex - 1];
    }
    assert alphaPt >= alphaLow - epsilon && alphaPt <= alphaHigh + epsilon : alphaPt;
    alphaPt = Math.max(alphaPt, alphaLow);
    alphaPt = Math.min(alphaPt, alphaHigh);
    log.trace("Sampled alpha {}", alphaPt);

    return alphaPt;
  }
Esempio n. 8
0
  public void calcscores() {
    Matrix mbase, mbaseavg;
    Matrix mtemp;
    double sco, div;
    @SuppressWarnings("unused")
    double scodef, divdef;
    mbase = clustinit.vtestsm;
    mbaseavg = clustinit.avgsm;
    scorevar = new double[(int) mbase.getRowCount()];
    scorestabdet = new double[(int) mbase.getRowCount()];
    scorevardef = new double[(int) mbase.getRowCount()];
    scorestabdesc = 0;
    scorestabpop = 0;
    for (int i = 0; i < 3; i++) {
      scorevar[i] = 0;
      scorevardef[i] = 0;
    }
    for (int i = 3; i < mbase.getRowCount(); i++) {

      mtemp = mbase.selectRows(Ret.LINK, i);
      sco = mtemp.abs(Ret.LINK).getMeanValue();
      sco = sco * mbaseavg.selectRows(Ret.LINK, i).getStdValue();
      div = mbaseavg.abs(Ret.LINK).selectRows(Ret.LINK, i).getMeanValue();
      if (div > 0) {
        sco = sco / div;
      } else {
        sco = 0;
      }
      scorevar[i] = sco;
    }
    mbase = clustinit.vtestsmdef;
    mbaseavg = clustinit.avgsmdef;
    for (int i = 3; i < mbase.getRowCount(); i++) {
      mtemp = mbase.selectRows(Ret.LINK, i);
      sco = mtemp.abs(Ret.LINK).getMeanValue();
      sco = sco * mbaseavg.selectRows(Ret.LINK, i).getStdValue();
      div = mbaseavg.abs(Ret.LINK).selectRows(Ret.LINK, i).getMeanValue();
      if (div > 0) {
        sco = sco / div;
      } else {
        sco = 0;
      }
      scorevardef[i] = sco;
    }
    scorestabdesc = 0;
    @SuppressWarnings("unused")
    int nbvarninit;
    for (int i = 3; i < mbase.getRowCount(); i++) {
      scorestabdet[i] = 0;
      for (int j = 0; j < mbase.getColumnCount(); j++) {
        scorestabdet[i] +=
            Math.abs(
                    clustinit.vtestsm.getAsDouble(i, j)
                        - clustinit.vtestsm.getAsDouble(i, clustinit.idtickinit))
                / Math.max(
                    Math.abs(clustinit.vtestsm.getAsDouble(i, j)),
                    Math.abs(clustinit.vtestsm.getAsDouble(i, clustinit.idtickinit)))
                / mbase.getColumnCount();
      }
      scorestabdet[i] = 1 - scorestabdet[i];
      if (!Double.isNaN(scorestabdet[i])) {
        scorestabdesc = scorestabdesc + scorestabdet[i] / (mbase.getRowCount() - 3);
      }
    }
    scorestabpop = 0;
    Cluster nc;
    for (int j = 0; j < clustHistDef.size(); j++) {
      nc = clustHistDef.get(j);
      scorestabpop +=
          (double) clustinit.getNumberOfCommonComponents(nc)
              / (double) Math.max(clustinit.getComponentIds().size(), nc.getComponentIds().size());
    }
    scorestabpop = (double) scorestabpop / clustinit.vtestsm.getColumnCount();

    long idColumn = mbase.getRowForLabel(Cluster.ID_C_NAME);
    long classLabelColumn = mbase.getRowForLabel(Cluster.CLASS_LABEL_C_NAME);
    long labelColorColumn = mbase.getRowForLabel("LABEL-COLOR");
    for (int i = 3; i < mbase.getRowCount(); i++) {
      Pattern p = Pattern.compile("CLASS_LABEL");
      Matcher m = p.matcher(mbase.getRowLabel(i));
      if ((i == idColumn || i == classLabelColumn || i == labelColorColumn || m.lookingAt())) {
        scorevardef[i] = 0;
        scorevar[i] = 0;
      }
    }
  }