public Long call() throws Exception {
Matrix newColumn = replaceInColumn(getSource(), bestGuess, column);
synchronized (imputed) {
for (int r = 0; r < newColumn.getRowCount(); r++) {
imputed.setAsDouble(newColumn.getAsDouble(r, 0), r, column);
}
}
return column;
}
public double getDouble(long... coordinates) {
if (inv == null) {
DenseMatrix A = new MTJDenseDoubleMatrix2D(getSource()).getWrappedObject();
DenseMatrix I = Matrices.identity((int) getSource().getColumnCount());
DenseMatrix AI = I.copy();
inv = new MTJDenseDoubleMatrix2D((DenseMatrix) A.solve(I, AI));
}
return inv.getAsDouble(coordinates);
}
public double getDouble(long... coordinates) {
if (mean == null) {
mean = new Mean(getDimension(), true, getSource()).calcNew();
}
double v = getSource().getAsDouble(coordinates);
if (MathUtil.isNaNOrInfinite(v)) {
switch (getDimension()) {
case ALL:
return mean.getAsDouble(0, 0);
case ROW:
return mean.getAsDouble(0, coordinates[COLUMN]);
case COLUMN:
return mean.getAsDouble(coordinates[ROW], 0);
}
} else {
return v;
}
return 0.0;
}
public double getDouble(long... coordinates) {
if (imputed == null) {
createMatrix();
}
double v = getSource().getAsDouble(coordinates);
if (MathUtil.isNaNOrInfinite(v)) {
return imputed.getAsDouble(coordinates);
} else {
return v;
}
}
private static Matrix replaceInColumn(Matrix original, Matrix firstGuess, long column) {
Matrix x = firstGuess.deleteColumns(Ret.NEW, column);
Matrix y = original.selectColumns(Ret.NEW, column);
List<Long> missingRows = new ArrayList<Long>();
for (long i = y.getRowCount(); --i >= 0; ) {
double v = y.getAsDouble(i, 0);
if (MathUtil.isNaNOrInfinite(v)) {
missingRows.add(i);
}
}
if (missingRows.isEmpty()) {
return y;
}
Matrix xdel = x.deleteRows(Ret.NEW, missingRows);
DenseDoubleMatrix2D bias1 = DenseDoubleMatrix2D.Factory.ones(xdel.getRowCount(), 1);
Matrix xtrain = Matrix.Factory.horCat(xdel, bias1);
Matrix ytrain = y.deleteRows(Ret.NEW, missingRows);
Matrix xinv = xtrain.pinv();
Matrix b = xinv.mtimes(ytrain);
DenseDoubleMatrix2D bias2 = DenseDoubleMatrix2D.Factory.ones(x.getRowCount(), 1);
Matrix yPredicted = Matrix.Factory.horCat(x, bias2).mtimes(b);
// set non-missing values back to original values
for (int row = 0; row < y.getRowCount(); row++) {
double v = y.getAsDouble(row, 0);
if (!Double.isNaN(v)) {
yPredicted.setAsDouble(v, row, 0);
}
}
return yPredicted;
}
@Override
protected double sampleAlpha(
Matrix direction, Matrix Aobj, Matrix objConst, double alphaLow, double alphaHigh) {
// Construct distribution over the randomly sampled line given by the direction and the bounds
// [alphaLow, alphaHigh]
Matrix AobjD = Aobj.mtimes(direction);
Matrix AobjP = Aobj.mtimes(currentPt);
OpenLongObjectHashMap alphaMap = new OpenLongObjectHashMap();
long alphaLowBin = Math.round(alphaLow * alphaRoundingScheme);
ObjFunHandle alphaLowHandle = ObjFunHandle.getHandle(alphaLowBin, alphaMap);
for (int i = 0; i < Aobj.getRowCount(); i++) {
double rate = AobjD.getAsDouble(i, 0);
double constant = AobjP.getAsDouble(i, 0) + objConst.getAsDouble(i, 0);
double alpha = (-constant) / rate;
long alphaBin = Math.round(alpha * alphaRoundingScheme);
log.trace("Alpha {}", alpha);
log.trace("Rate {} Constant {}", rate, constant);
if (rate > epsilon && alpha < alphaHigh) {
ObjFunHandle h;
if (alpha <= alphaLow) {
// This evidence is active over the entire interval
h = alphaLowHandle;
} else {
// This evidence will get active once we reach alpha in the interval
h = ObjFunHandle.getHandle(alphaBin, alphaMap);
}
h.increaseBy(rate, constant);
} else if (rate < -epsilon && alpha > alphaLow) {
alphaLowHandle.increaseBy(rate, constant);
if (alpha >= alphaHigh) {
// This evidence is active over the entire interval
} else {
// This evidence will get inactive once we reach alpha in the interval
ObjFunHandle h = ObjFunHandle.getHandle(alphaBin, alphaMap);
h.increaseBy(-rate, -constant);
}
} else { // Considered 0
alphaLowHandle.increaseBy(0.0, constant);
}
}
ObjFunHandle.getHandle(Math.round(alphaHigh * alphaRoundingScheme), alphaMap);
AobjD = null;
AobjP = null;
int length = alphaMap.size();
LongArrayList alphaBinsL = new LongArrayList(length);
ObjectArrayList handlesL = new ObjectArrayList(length);
alphaMap.pairsSortedByKey(alphaBinsL, handlesL);
assert alphaBinsL.size() == length && handlesL.size() == length;
double[] logcumulative = new double[length];
double[] cumRate = new double[length - 1];
double[] cumConst = new double[length - 1];
long[] alphaBins = alphaBinsL.elements();
Object[] handles = handlesL.elements();
log.trace("alphaBins: {}", Arrays.toString(alphaBins));
log.trace("handles: {}", Arrays.toString(handles));
// Now compute the cumulative probability distribution
ObjFunHandle base = (ObjFunHandle) handles[0];
double baserate = base.getRate();
double baseconstant = base.getConstant();
double basealpha = alphaBins[0] / (double) alphaRoundingScheme;
double baselogdivisor = baserate * basealpha + baseconstant;
logcumulative[0] = 0.0;
double rate = 0.0;
double constant = 0.0;
for (int i = 1; i < length; i++) {
ObjFunHandle h = (ObjFunHandle) handles[i - 1];
rate += h.getRate();
cumRate[i - 1] = rate;
constant += h.getConstant();
cumConst[i - 1] = constant;
double aL = alphaBins[i - 1] / (double) alphaRoundingScheme,
aU = alphaBins[i] / (double) alphaRoundingScheme;
double increment;
if (rate < -epsilon || rate > epsilon)
increment =
1.0
/ rate
* (Math.exp(baselogdivisor - rate * aL - constant)
- Math.exp(baselogdivisor - rate * aU - constant));
else increment = (aU - aL) * Math.exp(baselogdivisor - constant);
assert increment > -epsilon : increment;
logcumulative[i] = logcumulative[i - 1] + increment;
}
double total = logcumulative[length - 1];
double randomPt = Math.random() * total;
log.trace("Random Pt {}", randomPt);
log.trace("logcumulative: {}", Arrays.toString(logcumulative));
int findIndex = Arrays.binarySearch(logcumulative, randomPt);
log.trace("Find Index {}", findIndex);
double alphaPt;
if (findIndex >= 0) alphaPt = alphaBins[findIndex] / (double) alphaRoundingScheme;
else {
findIndex = -findIndex - 1;
assert findIndex > 0 : findIndex;
double aL = alphaBins[findIndex - 1] / (double) alphaRoundingScheme;
final double crate = cumRate[findIndex - 1];
final double cconstant = cumConst[findIndex - 1];
log.trace("R {}, c {} aL " + aL, crate, cconstant);
if (crate < -epsilon || crate > epsilon) {
double targetValue =
-randomPt
+ logcumulative[findIndex - 1]
+ 1.0 / crate * Math.exp(-cconstant - crate * aL + baselogdivisor);
log.trace("Target value {}", targetValue);
log.trace("Log: {}, inside log {}", Math.log(crate * targetValue), crate * targetValue);
alphaPt = -1.0 / crate * (Math.log(crate * targetValue) + cconstant - baselogdivisor);
} else {
double kcons = Math.exp(baselogdivisor - cconstant);
alphaPt = randomPt / kcons - logcumulative[findIndex - 1] / kcons + aL;
}
assert !Double.isNaN(alphaPt)
: "C: "
+ cconstant
+ " | R: "
+ crate
+ " Pt: "
+ randomPt
+ " aL: "
+ aL
+ " C0:"
+ logcumulative[findIndex - 1];
}
assert alphaPt >= alphaLow - epsilon && alphaPt <= alphaHigh + epsilon : alphaPt;
alphaPt = Math.max(alphaPt, alphaLow);
alphaPt = Math.min(alphaPt, alphaHigh);
log.trace("Sampled alpha {}", alphaPt);
return alphaPt;
}