public Map<String, Object> calculateObjects(Map<String, Object> input) { Map<String, Object> result = new HashMap<String, Object>(); Matrix source = MathUtil.getMatrix(input.get(SOURCE)); Matrix target = Matrix.Factory.zeros(source.getRowCount(), 1); long cols = source.getColumnCount(); long rows = source.getRowCount(); for (int k = 0; k < rows; k++) { int tp = 0; int fn = 0; for (int r = 0; r < rows; r++) { for (int c = 0; c < cols; c++) { int count = source.getAsInt(r, c); boolean expected = r == k; boolean predicted = c == k; if (expected && predicted) { tp += count; } else if (expected && (!predicted)) { fn += count; } } } target.setAsDouble(MathUtil.sensitivity(tp, fn), k, 0); } result.put(TARGET, target); return result; }
@Override public Map<Object, Object> calculateObjects(Map<Object, Object> input) throws MatrixException { Map<Object, Object> result = new HashMap<Object, Object>(); Matrix source1 = MathUtil.getMatrix(input.get(SOURCE1)); Matrix source2 = MathUtil.getMatrix(input.get(SOURCE2)); result.put(TARGET, source1.mtimes(source2)); return result; }
private boolean containsMissingValues(long c) { for (int r = 0; r < getSource().getRowCount(); r++) { if (MathUtil.isNaNOrInfinite(getSource().getAsDouble(r, c))) { return true; } } return false; }
public Map<String, Object> calculateObjects(Map<String, Object> input) { int dimension = defaultDimension; boolean ignoreNaN = defaultIgnoreNaN; Map<String, Object> result = new HashMap<String, Object>(); Matrix source = MathUtil.getMatrix(input.get(SOURCE)); Object o2 = input.get(DIMENSION); if (o2 != null) { dimension = MathUtil.getInt(o2); } Object o3 = input.get(IGNORENAN); if (o3 != null) { ignoreNaN = MathUtil.getBoolean(o3); } Matrix target = source.mean(Ret.NEW, dimension, ignoreNaN); result.put(TARGET, target); return result; }
public double getDouble(long... coordinates) { if (imputed == null) { createMatrix(); } double v = getSource().getAsDouble(coordinates); if (MathUtil.isNaNOrInfinite(v)) { return imputed.getAsDouble(coordinates); } else { return v; } }
@SuppressWarnings("unchecked") public final void setAsObject(Object o, long... coordinates) { switch (getValueType()) { case BOOLEAN: setAsBoolean(MathUtil.getBoolean(o), coordinates); break; case BIGINTEGER: setAsBigInteger(MathUtil.getBigInteger(o), coordinates); break; case BIGDECIMAL: setAsBigDecimal(MathUtil.getBigDecimal(o), coordinates); break; case BYTE: setAsByte(MathUtil.getByte(o), coordinates); break; case CHAR: setAsChar(MathUtil.getChar(o), coordinates); break; case DOUBLE: setAsDouble(MathUtil.getDouble(o), coordinates); break; case FLOAT: setAsFloat(MathUtil.getFloat(o), coordinates); break; case INT: setAsInt(MathUtil.getInt(o), coordinates); break; case LONG: setAsLong(MathUtil.getLong(o), coordinates); break; case OBJECT: setObject((T) o, coordinates); break; case SHORT: setAsShort(MathUtil.getShort(o), coordinates); break; case STRING: setAsString(StringUtil.convert(o), coordinates); break; } }
public double getDouble(long... coordinates) { if (mean == null) { mean = new Mean(getDimension(), true, getSource()).calcNew(); } double v = getSource().getAsDouble(coordinates); if (MathUtil.isNaNOrInfinite(v)) { switch (getDimension()) { case ALL: return mean.getAsDouble(0, 0); case ROW: return mean.getAsDouble(0, coordinates[COLUMN]); case COLUMN: return mean.getAsDouble(coordinates[ROW], 0); } } else { return v; } return 0.0; }
private static Matrix replaceInColumn(Matrix original, Matrix firstGuess, long column) { Matrix x = firstGuess.deleteColumns(Ret.NEW, column); Matrix y = original.selectColumns(Ret.NEW, column); List<Long> missingRows = new ArrayList<Long>(); for (long i = y.getRowCount(); --i >= 0; ) { double v = y.getAsDouble(i, 0); if (MathUtil.isNaNOrInfinite(v)) { missingRows.add(i); } } if (missingRows.isEmpty()) { return y; } Matrix xdel = x.deleteRows(Ret.NEW, missingRows); DenseDoubleMatrix2D bias1 = DenseDoubleMatrix2D.Factory.ones(xdel.getRowCount(), 1); Matrix xtrain = Matrix.Factory.horCat(xdel, bias1); Matrix ytrain = y.deleteRows(Ret.NEW, missingRows); Matrix xinv = xtrain.pinv(); Matrix b = xinv.mtimes(ytrain); DenseDoubleMatrix2D bias2 = DenseDoubleMatrix2D.Factory.ones(x.getRowCount(), 1); Matrix yPredicted = Matrix.Factory.horCat(x, bias2).mtimes(b); // set non-missing values back to original values for (int row = 0; row < y.getRowCount(); row++) { double v = y.getAsDouble(row, 0); if (!Double.isNaN(v)) { yPredicted.setAsDouble(v, row, 0); } } return yPredicted; }
private void createMatrix() { try { ExecutorService executor = Executors.newFixedThreadPool(UJMPSettings.getInstance().getNumberOfThreads()); Matrix x = getSource(); double valueCount = x.getValueCount(); long missingCount = (long) x.countMissing(Ret.NEW, Matrix.ALL).getEuklideanValue(); double percent = ((int) Math.round((missingCount * 1000.0 / valueCount))) / 10.0; System.out.println("missing values: " + missingCount + " (" + percent + "%)"); System.out.println("============================================"); if (bestGuess == null) { bestGuess = getSource().impute(Ret.NEW, ImputationMethod.RowMean); } int run = 0; double d; do { System.out.println("Iteration " + run++); List<Future<Long>> futures = new ArrayList<Future<Long>>(); imputed = Matrix.Factory.zeros(x.getSize()); long t0 = System.currentTimeMillis(); for (long c = 0; c < x.getColumnCount(); c++) { if (containsMissingValues(c)) { futures.add(executor.submit(new PredictColumn(c))); } } for (Future<Long> f : futures) { Long completedCols = f.get(); long elapsedTime = System.currentTimeMillis() - t0; long remainingCols = x.getColumnCount() - completedCols; double colsPerMillisecond = (double) (completedCols + 1) / (double) elapsedTime; long remainingTime = (long) (remainingCols / colsPerMillisecond / 1000.0); System.out.println( (completedCols * 1000 / x.getColumnCount() / 10.0) + "% completed (" + remainingTime + " seconds remaining)"); } Matrix newBestGuess = bestGuess.times(decay).plus(imputed.times(1 - decay)); for (int r = 0; r < getSource().getRowCount(); r++) { for (int c = 0; c < getSource().getColumnCount(); c++) { double value = getSource().getAsDouble(r, c); if (!MathUtil.isNaNOrInfinite(value)) { newBestGuess.setAsDouble(value, r, c); } } } d = newBestGuess.euklideanDistanceTo(bestGuess, true) / missingCount; System.out.println("delta: " + d); System.out.println("============================================"); bestGuess = newBestGuess; bestGuess.exportTo().file(tempFile).asDenseCSV(); } while (delta < d); executor.shutdown(); imputed = bestGuess; if (imputed.containsMissingValues()) { throw new RuntimeException("Matrix has still missing values after imputation"); } } catch (Exception e) { throw new RuntimeException(e); } }
public boolean getBoolean(long... coordinates) throws MatrixException { return MathUtil.xor( getSource().getAsBoolean(coordinates), getSources()[1].getAsBoolean(coordinates)); }