public Map<String, Object> calculateObjects(Map<String, Object> input) {
Map<String, Object> result = new HashMap<String, Object>();
Matrix source = MathUtil.getMatrix(input.get(SOURCE));
Matrix target = Matrix.Factory.zeros(source.getRowCount(), 1);
long cols = source.getColumnCount();
long rows = source.getRowCount();
for (int k = 0; k < rows; k++) {
int tp = 0;
int fn = 0;
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
int count = source.getAsInt(r, c);
boolean expected = r == k;
boolean predicted = c == k;
if (expected && predicted) {
tp += count;
} else if (expected && (!predicted)) {
fn += count;
}
}
}
target.setAsDouble(MathUtil.sensitivity(tp, fn), k, 0);
}
result.put(TARGET, target);
return result;
}
private static Matrix replaceInColumn(Matrix original, Matrix firstGuess, long column) {
Matrix x = firstGuess.deleteColumns(Ret.NEW, column);
Matrix y = original.selectColumns(Ret.NEW, column);
List<Long> missingRows = new ArrayList<Long>();
for (long i = y.getRowCount(); --i >= 0; ) {
double v = y.getAsDouble(i, 0);
if (MathUtil.isNaNOrInfinite(v)) {
missingRows.add(i);
}
}
if (missingRows.isEmpty()) {
return y;
}
Matrix xdel = x.deleteRows(Ret.NEW, missingRows);
DenseDoubleMatrix2D bias1 = DenseDoubleMatrix2D.Factory.ones(xdel.getRowCount(), 1);
Matrix xtrain = Matrix.Factory.horCat(xdel, bias1);
Matrix ytrain = y.deleteRows(Ret.NEW, missingRows);
Matrix xinv = xtrain.pinv();
Matrix b = xinv.mtimes(ytrain);
DenseDoubleMatrix2D bias2 = DenseDoubleMatrix2D.Factory.ones(x.getRowCount(), 1);
Matrix yPredicted = Matrix.Factory.horCat(x, bias2).mtimes(b);
// set non-missing values back to original values
for (int row = 0; row < y.getRowCount(); row++) {
double v = y.getAsDouble(row, 0);
if (!Double.isNaN(v)) {
yPredicted.setAsDouble(v, row, 0);
}
}
return yPredicted;
}
private void createMatrix() {
try {
ExecutorService executor =
Executors.newFixedThreadPool(UJMPSettings.getInstance().getNumberOfThreads());
Matrix x = getSource();
double valueCount = x.getValueCount();
long missingCount = (long) x.countMissing(Ret.NEW, Matrix.ALL).getEuklideanValue();
double percent = ((int) Math.round((missingCount * 1000.0 / valueCount))) / 10.0;
System.out.println("missing values: " + missingCount + " (" + percent + "%)");
System.out.println("============================================");
if (bestGuess == null) {
bestGuess = getSource().impute(Ret.NEW, ImputationMethod.RowMean);
}
int run = 0;
double d;
do {
System.out.println("Iteration " + run++);
List<Future<Long>> futures = new ArrayList<Future<Long>>();
imputed = Matrix.Factory.zeros(x.getSize());
long t0 = System.currentTimeMillis();
for (long c = 0; c < x.getColumnCount(); c++) {
if (containsMissingValues(c)) {
futures.add(executor.submit(new PredictColumn(c)));
}
}
for (Future<Long> f : futures) {
Long completedCols = f.get();
long elapsedTime = System.currentTimeMillis() - t0;
long remainingCols = x.getColumnCount() - completedCols;
double colsPerMillisecond = (double) (completedCols + 1) / (double) elapsedTime;
long remainingTime = (long) (remainingCols / colsPerMillisecond / 1000.0);
System.out.println(
(completedCols * 1000 / x.getColumnCount() / 10.0)
+ "% completed ("
+ remainingTime
+ " seconds remaining)");
}
Matrix newBestGuess = bestGuess.times(decay).plus(imputed.times(1 - decay));
for (int r = 0; r < getSource().getRowCount(); r++) {
for (int c = 0; c < getSource().getColumnCount(); c++) {
double value = getSource().getAsDouble(r, c);
if (!MathUtil.isNaNOrInfinite(value)) {
newBestGuess.setAsDouble(value, r, c);
}
}
}
d = newBestGuess.euklideanDistanceTo(bestGuess, true) / missingCount;
System.out.println("delta: " + d);
System.out.println("============================================");
bestGuess = newBestGuess;
bestGuess.exportTo().file(tempFile).asDenseCSV();
} while (delta < d);
executor.shutdown();
imputed = bestGuess;
if (imputed.containsMissingValues()) {
throw new RuntimeException("Matrix has still missing values after imputation");
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
