public Map<String, Object> calculateObjects(Map<String, Object> input) {
Map<String, Object> result = new HashMap<String, Object>();
Matrix source = MathUtil.getMatrix(input.get(SOURCE));
Matrix target = Matrix.Factory.zeros(source.getRowCount(), 1);
long cols = source.getColumnCount();
long rows = source.getRowCount();
for (int k = 0; k < rows; k++) {
int tp = 0;
int fn = 0;
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
int count = source.getAsInt(r, c);
boolean expected = r == k;
boolean predicted = c == k;
if (expected && predicted) {
tp += count;
} else if (expected && (!predicted)) {
fn += count;
}
}
}
target.setAsDouble(MathUtil.sensitivity(tp, fn), k, 0);
}
result.put(TARGET, target);
return result;
}
@Override
public Map<Object, Object> calculateObjects(Map<Object, Object> input) throws MatrixException {
Map<Object, Object> result = new HashMap<Object, Object>();
Matrix source1 = MathUtil.getMatrix(input.get(SOURCE1));
Matrix source2 = MathUtil.getMatrix(input.get(SOURCE2));
result.put(TARGET, source1.mtimes(source2));
return result;
}
private boolean containsMissingValues(long c) {
for (int r = 0; r < getSource().getRowCount(); r++) {
if (MathUtil.isNaNOrInfinite(getSource().getAsDouble(r, c))) {
return true;
}
}
return false;
}
public Map<String, Object> calculateObjects(Map<String, Object> input) {
int dimension = defaultDimension;
boolean ignoreNaN = defaultIgnoreNaN;
Map<String, Object> result = new HashMap<String, Object>();
Matrix source = MathUtil.getMatrix(input.get(SOURCE));
Object o2 = input.get(DIMENSION);
if (o2 != null) {
dimension = MathUtil.getInt(o2);
}
Object o3 = input.get(IGNORENAN);
if (o3 != null) {
ignoreNaN = MathUtil.getBoolean(o3);
}
Matrix target = source.mean(Ret.NEW, dimension, ignoreNaN);
result.put(TARGET, target);
return result;
}
public double getDouble(long... coordinates) {
if (imputed == null) {
createMatrix();
}
double v = getSource().getAsDouble(coordinates);
if (MathUtil.isNaNOrInfinite(v)) {
return imputed.getAsDouble(coordinates);
} else {
return v;
}
}
@SuppressWarnings("unchecked")
public final void setAsObject(Object o, long... coordinates) {
switch (getValueType()) {
case BOOLEAN:
setAsBoolean(MathUtil.getBoolean(o), coordinates);
break;
case BIGINTEGER:
setAsBigInteger(MathUtil.getBigInteger(o), coordinates);
break;
case BIGDECIMAL:
setAsBigDecimal(MathUtil.getBigDecimal(o), coordinates);
break;
case BYTE:
setAsByte(MathUtil.getByte(o), coordinates);
break;
case CHAR:
setAsChar(MathUtil.getChar(o), coordinates);
break;
case DOUBLE:
setAsDouble(MathUtil.getDouble(o), coordinates);
break;
case FLOAT:
setAsFloat(MathUtil.getFloat(o), coordinates);
break;
case INT:
setAsInt(MathUtil.getInt(o), coordinates);
break;
case LONG:
setAsLong(MathUtil.getLong(o), coordinates);
break;
case OBJECT:
setObject((T) o, coordinates);
break;
case SHORT:
setAsShort(MathUtil.getShort(o), coordinates);
break;
case STRING:
setAsString(StringUtil.convert(o), coordinates);
break;
}
}
public double getDouble(long... coordinates) {
if (mean == null) {
mean = new Mean(getDimension(), true, getSource()).calcNew();
}
double v = getSource().getAsDouble(coordinates);
if (MathUtil.isNaNOrInfinite(v)) {
switch (getDimension()) {
case ALL:
return mean.getAsDouble(0, 0);
case ROW:
return mean.getAsDouble(0, coordinates[COLUMN]);
case COLUMN:
return mean.getAsDouble(coordinates[ROW], 0);
}
} else {
return v;
}
return 0.0;
}
private static Matrix replaceInColumn(Matrix original, Matrix firstGuess, long column) {
Matrix x = firstGuess.deleteColumns(Ret.NEW, column);
Matrix y = original.selectColumns(Ret.NEW, column);
List<Long> missingRows = new ArrayList<Long>();
for (long i = y.getRowCount(); --i >= 0; ) {
double v = y.getAsDouble(i, 0);
if (MathUtil.isNaNOrInfinite(v)) {
missingRows.add(i);
}
}
if (missingRows.isEmpty()) {
return y;
}
Matrix xdel = x.deleteRows(Ret.NEW, missingRows);
DenseDoubleMatrix2D bias1 = DenseDoubleMatrix2D.Factory.ones(xdel.getRowCount(), 1);
Matrix xtrain = Matrix.Factory.horCat(xdel, bias1);
Matrix ytrain = y.deleteRows(Ret.NEW, missingRows);
Matrix xinv = xtrain.pinv();
Matrix b = xinv.mtimes(ytrain);
DenseDoubleMatrix2D bias2 = DenseDoubleMatrix2D.Factory.ones(x.getRowCount(), 1);
Matrix yPredicted = Matrix.Factory.horCat(x, bias2).mtimes(b);
// set non-missing values back to original values
for (int row = 0; row < y.getRowCount(); row++) {
double v = y.getAsDouble(row, 0);
if (!Double.isNaN(v)) {
yPredicted.setAsDouble(v, row, 0);
}
}
return yPredicted;
}
private void createMatrix() {
try {
ExecutorService executor =
Executors.newFixedThreadPool(UJMPSettings.getInstance().getNumberOfThreads());
Matrix x = getSource();
double valueCount = x.getValueCount();
long missingCount = (long) x.countMissing(Ret.NEW, Matrix.ALL).getEuklideanValue();
double percent = ((int) Math.round((missingCount * 1000.0 / valueCount))) / 10.0;
System.out.println("missing values: " + missingCount + " (" + percent + "%)");
System.out.println("============================================");
if (bestGuess == null) {
bestGuess = getSource().impute(Ret.NEW, ImputationMethod.RowMean);
}
int run = 0;
double d;
do {
System.out.println("Iteration " + run++);
List<Future<Long>> futures = new ArrayList<Future<Long>>();
imputed = Matrix.Factory.zeros(x.getSize());
long t0 = System.currentTimeMillis();
for (long c = 0; c < x.getColumnCount(); c++) {
if (containsMissingValues(c)) {
futures.add(executor.submit(new PredictColumn(c)));
}
}
for (Future<Long> f : futures) {
Long completedCols = f.get();
long elapsedTime = System.currentTimeMillis() - t0;
long remainingCols = x.getColumnCount() - completedCols;
double colsPerMillisecond = (double) (completedCols + 1) / (double) elapsedTime;
long remainingTime = (long) (remainingCols / colsPerMillisecond / 1000.0);
System.out.println(
(completedCols * 1000 / x.getColumnCount() / 10.0)
+ "% completed ("
+ remainingTime
+ " seconds remaining)");
}
Matrix newBestGuess = bestGuess.times(decay).plus(imputed.times(1 - decay));
for (int r = 0; r < getSource().getRowCount(); r++) {
for (int c = 0; c < getSource().getColumnCount(); c++) {
double value = getSource().getAsDouble(r, c);
if (!MathUtil.isNaNOrInfinite(value)) {
newBestGuess.setAsDouble(value, r, c);
}
}
}
d = newBestGuess.euklideanDistanceTo(bestGuess, true) / missingCount;
System.out.println("delta: " + d);
System.out.println("============================================");
bestGuess = newBestGuess;
bestGuess.exportTo().file(tempFile).asDenseCSV();
} while (delta < d);
executor.shutdown();
imputed = bestGuess;
if (imputed.containsMissingValues()) {
throw new RuntimeException("Matrix has still missing values after imputation");
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public boolean getBoolean(long... coordinates) throws MatrixException {
return MathUtil.xor(
getSource().getAsBoolean(coordinates), getSources()[1].getAsBoolean(coordinates));
}