public Map<String, Object> calculateObjects(Map<String, Object> input) {
Map<String, Object> result = new HashMap<String, Object>();
Matrix source = MathUtil.getMatrix(input.get(SOURCE));
Matrix target = Matrix.Factory.zeros(source.getRowCount(), 1);
long cols = source.getColumnCount();
long rows = source.getRowCount();
for (int k = 0; k < rows; k++) {
int tp = 0;
int fn = 0;
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
int count = source.getAsInt(r, c);
boolean expected = r == k;
boolean predicted = c == k;
if (expected && predicted) {
tp += count;
} else if (expected && (!predicted)) {
fn += count;
}
}
}
target.setAsDouble(MathUtil.sensitivity(tp, fn), k, 0);
}
result.put(TARGET, target);
return result;
}
public Long call() throws Exception {
Matrix newColumn = replaceInColumn(getSource(), bestGuess, column);
synchronized (imputed) {
for (int r = 0; r < newColumn.getRowCount(); r++) {
imputed.setAsDouble(newColumn.getAsDouble(r, 0), r, column);
}
}
return column;
}
private Matrix calculate() {
wordMapping = new DefaultMapMatrix<String, Long>();
Matrix m = getSource();
for (long[] c : m.availableCoordinates()) {
String s = m.getAsString(c);
if (s != null) {
String[] words = s.split("\\s+");
for (String w : words) {
if (w.length() == 0) {
continue;
}
Long i = wordMapping.get(w);
if (i == null) {
wordMapping.put(w, wordMapping.getRowCount());
}
}
}
}
result = new DefaultSparseLongMatrix(m.getRowCount(), wordMapping.getRowCount());
long rowCount = m.getRowCount();
long colCount = m.getColumnCount();
for (long row = 0; row < rowCount; row++) {
for (long col = 0; col < colCount; col++) {
String string = m.getAsString(row, col);
if (string != null && string.length() > 0) {
String[] words = string.split("[\\s]+");
for (String w : words) {
if (w.length() == 0) {
continue;
}
long i = wordMapping.get(w);
int count = result.getAsInt(row, i);
result.setAsInt(++count, row, i);
}
}
}
}
return result;
}
private static Matrix replaceInColumn(Matrix original, Matrix firstGuess, long column) {
Matrix x = firstGuess.deleteColumns(Ret.NEW, column);
Matrix y = original.selectColumns(Ret.NEW, column);
List<Long> missingRows = new ArrayList<Long>();
for (long i = y.getRowCount(); --i >= 0; ) {
double v = y.getAsDouble(i, 0);
if (MathUtil.isNaNOrInfinite(v)) {
missingRows.add(i);
}
}
if (missingRows.isEmpty()) {
return y;
}
Matrix xdel = x.deleteRows(Ret.NEW, missingRows);
DenseDoubleMatrix2D bias1 = DenseDoubleMatrix2D.Factory.ones(xdel.getRowCount(), 1);
Matrix xtrain = Matrix.Factory.horCat(xdel, bias1);
Matrix ytrain = y.deleteRows(Ret.NEW, missingRows);
Matrix xinv = xtrain.pinv();
Matrix b = xinv.mtimes(ytrain);
DenseDoubleMatrix2D bias2 = DenseDoubleMatrix2D.Factory.ones(x.getRowCount(), 1);
Matrix yPredicted = Matrix.Factory.horCat(x, bias2).mtimes(b);
// set non-missing values back to original values
for (int row = 0; row < y.getRowCount(); row++) {
double v = y.getAsDouble(row, 0);
if (!Double.isNaN(v)) {
yPredicted.setAsDouble(v, row, 0);
}
}
return yPredicted;
}
public DefaultDenseObjectMatrix2D(Matrix m) throws MatrixException {
this.rows = (int) m.getRowCount();
this.cols = (int) m.getColumnCount();
this.size = new long[] {rows, cols};
if (m instanceof DefaultDenseObjectMatrix2D) {
Object[] v = ((DefaultDenseObjectMatrix2D) m).values;
this.values = new Object[v.length];
System.arraycopy(v, 0, this.values, 0, v.length);
} else {
this.values = new Object[rows * cols];
for (long[] c : m.allCoordinates()) {
setObject(m.getAsObject(c), c);
}
}
}
public void asLatex() throws IOException {
final String EOL = System.getProperty("line.separator");
final Writer writer = getWriter();
final Matrix matrix = getMatrix();
final long rowCount = matrix.getRowCount();
final long colCount = matrix.getColumnCount();
writer.write("\\begin{table}[!ht]" + EOL);
writer.write("\\centering" + EOL);
if (matrix.getLabelObject() != null) {
writer.write(
"\\caption{"
+ UJMPFormat.getSingleLineInstance().format(matrix.getLabelObject())
+ "}"
+ EOL);
}
StringBuilder buf = new StringBuilder();
for (long i = matrix.getColumnCount() - 1; i != -1; i--) {
buf.append('c');
}
String alignment = buf.toString();
writer.write("\\begin{tabular}{" + alignment + "}" + EOL);
writer.write("\\toprule" + EOL);
for (int row = 0; row < rowCount; row++) {
for (int col = 0; col < colCount; col++) {
writer.write(UJMPFormat.getSingleLineInstance().format(matrix.getAsObject(row, col)));
if (col < colCount - 1) {
writer.write(" & ");
}
}
writer.write(" \\\\" + EOL);
}
writer.write("\\bottomrule" + EOL);
writer.write("\\end{tabular}" + EOL);
writer.write("\\end{table}" + EOL);
}
@Override
protected double sampleAlpha(
Matrix direction, Matrix Aobj, Matrix objConst, double alphaLow, double alphaHigh) {
// Construct distribution over the randomly sampled line given by the direction and the bounds
// [alphaLow, alphaHigh]
Matrix AobjD = Aobj.mtimes(direction);
Matrix AobjP = Aobj.mtimes(currentPt);
OpenLongObjectHashMap alphaMap = new OpenLongObjectHashMap();
long alphaLowBin = Math.round(alphaLow * alphaRoundingScheme);
ObjFunHandle alphaLowHandle = ObjFunHandle.getHandle(alphaLowBin, alphaMap);
for (int i = 0; i < Aobj.getRowCount(); i++) {
double rate = AobjD.getAsDouble(i, 0);
double constant = AobjP.getAsDouble(i, 0) + objConst.getAsDouble(i, 0);
double alpha = (-constant) / rate;
long alphaBin = Math.round(alpha * alphaRoundingScheme);
log.trace("Alpha {}", alpha);
log.trace("Rate {} Constant {}", rate, constant);
if (rate > epsilon && alpha < alphaHigh) {
ObjFunHandle h;
if (alpha <= alphaLow) {
// This evidence is active over the entire interval
h = alphaLowHandle;
} else {
// This evidence will get active once we reach alpha in the interval
h = ObjFunHandle.getHandle(alphaBin, alphaMap);
}
h.increaseBy(rate, constant);
} else if (rate < -epsilon && alpha > alphaLow) {
alphaLowHandle.increaseBy(rate, constant);
if (alpha >= alphaHigh) {
// This evidence is active over the entire interval
} else {
// This evidence will get inactive once we reach alpha in the interval
ObjFunHandle h = ObjFunHandle.getHandle(alphaBin, alphaMap);
h.increaseBy(-rate, -constant);
}
} else { // Considered 0
alphaLowHandle.increaseBy(0.0, constant);
}
}
ObjFunHandle.getHandle(Math.round(alphaHigh * alphaRoundingScheme), alphaMap);
AobjD = null;
AobjP = null;
int length = alphaMap.size();
LongArrayList alphaBinsL = new LongArrayList(length);
ObjectArrayList handlesL = new ObjectArrayList(length);
alphaMap.pairsSortedByKey(alphaBinsL, handlesL);
assert alphaBinsL.size() == length && handlesL.size() == length;
double[] logcumulative = new double[length];
double[] cumRate = new double[length - 1];
double[] cumConst = new double[length - 1];
long[] alphaBins = alphaBinsL.elements();
Object[] handles = handlesL.elements();
log.trace("alphaBins: {}", Arrays.toString(alphaBins));
log.trace("handles: {}", Arrays.toString(handles));
// Now compute the cumulative probability distribution
ObjFunHandle base = (ObjFunHandle) handles[0];
double baserate = base.getRate();
double baseconstant = base.getConstant();
double basealpha = alphaBins[0] / (double) alphaRoundingScheme;
double baselogdivisor = baserate * basealpha + baseconstant;
logcumulative[0] = 0.0;
double rate = 0.0;
double constant = 0.0;
for (int i = 1; i < length; i++) {
ObjFunHandle h = (ObjFunHandle) handles[i - 1];
rate += h.getRate();
cumRate[i - 1] = rate;
constant += h.getConstant();
cumConst[i - 1] = constant;
double aL = alphaBins[i - 1] / (double) alphaRoundingScheme,
aU = alphaBins[i] / (double) alphaRoundingScheme;
double increment;
if (rate < -epsilon || rate > epsilon)
increment =
1.0
/ rate
* (Math.exp(baselogdivisor - rate * aL - constant)
- Math.exp(baselogdivisor - rate * aU - constant));
else increment = (aU - aL) * Math.exp(baselogdivisor - constant);
assert increment > -epsilon : increment;
logcumulative[i] = logcumulative[i - 1] + increment;
}
double total = logcumulative[length - 1];
double randomPt = Math.random() * total;
log.trace("Random Pt {}", randomPt);
log.trace("logcumulative: {}", Arrays.toString(logcumulative));
int findIndex = Arrays.binarySearch(logcumulative, randomPt);
log.trace("Find Index {}", findIndex);
double alphaPt;
if (findIndex >= 0) alphaPt = alphaBins[findIndex] / (double) alphaRoundingScheme;
else {
findIndex = -findIndex - 1;
assert findIndex > 0 : findIndex;
double aL = alphaBins[findIndex - 1] / (double) alphaRoundingScheme;
final double crate = cumRate[findIndex - 1];
final double cconstant = cumConst[findIndex - 1];
log.trace("R {}, c {} aL " + aL, crate, cconstant);
if (crate < -epsilon || crate > epsilon) {
double targetValue =
-randomPt
+ logcumulative[findIndex - 1]
+ 1.0 / crate * Math.exp(-cconstant - crate * aL + baselogdivisor);
log.trace("Target value {}", targetValue);
log.trace("Log: {}, inside log {}", Math.log(crate * targetValue), crate * targetValue);
alphaPt = -1.0 / crate * (Math.log(crate * targetValue) + cconstant - baselogdivisor);
} else {
double kcons = Math.exp(baselogdivisor - cconstant);
alphaPt = randomPt / kcons - logcumulative[findIndex - 1] / kcons + aL;
}
assert !Double.isNaN(alphaPt)
: "C: "
+ cconstant
+ " | R: "
+ crate
+ " Pt: "
+ randomPt
+ " aL: "
+ aL
+ " C0:"
+ logcumulative[findIndex - 1];
}
assert alphaPt >= alphaLow - epsilon && alphaPt <= alphaHigh + epsilon : alphaPt;
alphaPt = Math.max(alphaPt, alphaLow);
alphaPt = Math.min(alphaPt, alphaHigh);
log.trace("Sampled alpha {}", alphaPt);
return alphaPt;
}
public void calcscores() {
Matrix mbase, mbaseavg;
Matrix mtemp;
double sco, div;
@SuppressWarnings("unused")
double scodef, divdef;
mbase = clustinit.vtestsm;
mbaseavg = clustinit.avgsm;
scorevar = new double[(int) mbase.getRowCount()];
scorestabdet = new double[(int) mbase.getRowCount()];
scorevardef = new double[(int) mbase.getRowCount()];
scorestabdesc = 0;
scorestabpop = 0;
for (int i = 0; i < 3; i++) {
scorevar[i] = 0;
scorevardef[i] = 0;
}
for (int i = 3; i < mbase.getRowCount(); i++) {
mtemp = mbase.selectRows(Ret.LINK, i);
sco = mtemp.abs(Ret.LINK).getMeanValue();
sco = sco * mbaseavg.selectRows(Ret.LINK, i).getStdValue();
div = mbaseavg.abs(Ret.LINK).selectRows(Ret.LINK, i).getMeanValue();
if (div > 0) {
sco = sco / div;
} else {
sco = 0;
}
scorevar[i] = sco;
}
mbase = clustinit.vtestsmdef;
mbaseavg = clustinit.avgsmdef;
for (int i = 3; i < mbase.getRowCount(); i++) {
mtemp = mbase.selectRows(Ret.LINK, i);
sco = mtemp.abs(Ret.LINK).getMeanValue();
sco = sco * mbaseavg.selectRows(Ret.LINK, i).getStdValue();
div = mbaseavg.abs(Ret.LINK).selectRows(Ret.LINK, i).getMeanValue();
if (div > 0) {
sco = sco / div;
} else {
sco = 0;
}
scorevardef[i] = sco;
}
scorestabdesc = 0;
@SuppressWarnings("unused")
int nbvarninit;
for (int i = 3; i < mbase.getRowCount(); i++) {
scorestabdet[i] = 0;
for (int j = 0; j < mbase.getColumnCount(); j++) {
scorestabdet[i] +=
Math.abs(
clustinit.vtestsm.getAsDouble(i, j)
- clustinit.vtestsm.getAsDouble(i, clustinit.idtickinit))
/ Math.max(
Math.abs(clustinit.vtestsm.getAsDouble(i, j)),
Math.abs(clustinit.vtestsm.getAsDouble(i, clustinit.idtickinit)))
/ mbase.getColumnCount();
}
scorestabdet[i] = 1 - scorestabdet[i];
if (!Double.isNaN(scorestabdet[i])) {
scorestabdesc = scorestabdesc + scorestabdet[i] / (mbase.getRowCount() - 3);
}
}
scorestabpop = 0;
Cluster nc;
for (int j = 0; j < clustHistDef.size(); j++) {
nc = clustHistDef.get(j);
scorestabpop +=
(double) clustinit.getNumberOfCommonComponents(nc)
/ (double) Math.max(clustinit.getComponentIds().size(), nc.getComponentIds().size());
}
scorestabpop = (double) scorestabpop / clustinit.vtestsm.getColumnCount();
long idColumn = mbase.getRowForLabel(Cluster.ID_C_NAME);
long classLabelColumn = mbase.getRowForLabel(Cluster.CLASS_LABEL_C_NAME);
long labelColorColumn = mbase.getRowForLabel("LABEL-COLOR");
for (int i = 3; i < mbase.getRowCount(); i++) {
Pattern p = Pattern.compile("CLASS_LABEL");
Matcher m = p.matcher(mbase.getRowLabel(i));
if ((i == idColumn || i == classLabelColumn || i == labelColorColumn || m.lookingAt())) {
scorevardef[i] = 0;
scorevar[i] = 0;
}
}
}