public Object call() throws MatrixException {
Matrix m =
getMatrixObject()
.getMatrix()
.bootstrap(getNewOrLink(), GUIUtil.getInt("Number of samples", 1, 10000000));
m.showGUI();
return m;
}
public final Matrix copy() throws MatrixException {
Object[] result = new Object[values.length];
System.arraycopy(values, 0, result, 0, values.length);
Matrix m = new DefaultDenseObjectMatrix2D(result, rows, cols);
if (getAnnotation() != null) {
m.setAnnotation(getAnnotation().clone());
}
return m;
}
public Long call() throws Exception {
Matrix newColumn = replaceInColumn(getSource(), bestGuess, column);
synchronized (imputed) {
for (int r = 0; r < newColumn.getRowCount(); r++) {
imputed.setAsDouble(newColumn.getAsDouble(r, 0), r, column);
}
}
return column;
}
@Override
public Map<Object, Object> calculateObjects(Map<Object, Object> input) throws MatrixException {
Map<Object, Object> result = new HashMap<Object, Object>();
Matrix source1 = MathUtil.getMatrix(input.get(SOURCE1));
Matrix source2 = MathUtil.getMatrix(input.get(SOURCE2));
result.put(TARGET, source1.mtimes(source2));
return result;
}
public Xor(Matrix m1, Matrix m2) {
super(m1, m2);
if (m2.isScalar() && !Coordinates.equals(m1.getSize(), m2.getSize())) {
getSources()[1] = MatrixFactory.fill(m2.getAsBoolean(0, 0), m1.getSize());
} else if (m1.isScalar() && !Coordinates.equals(m1.getSize(), m2.getSize())) {
getSources()[0] = MatrixFactory.fill(m1.getAsBoolean(0, 0), m2.getSize());
}
}
public DefaultDenseObjectMatrix2D(Matrix m) throws MatrixException {
this.rows = (int) m.getRowCount();
this.cols = (int) m.getColumnCount();
this.size = new long[] {rows, cols};
if (m instanceof DefaultDenseObjectMatrix2D) {
Object[] v = ((DefaultDenseObjectMatrix2D) m).values;
this.values = new Object[v.length];
System.arraycopy(v, 0, this.values, 0, v.length);
} else {
this.values = new Object[rows * cols];
for (long[] c : m.allCoordinates()) {
setObject(m.getAsObject(c), c);
}
}
}
public Object call() {
try {
String s =
JOptionPane.showInputDialog(
getComponent(),
"Enter the rows and columns to select, e.g. 1,3-5;4-5,7",
"Select",
JOptionPane.QUESTION_MESSAGE);
Matrix m = getMatrixObject().getMatrix().select(getNewOrLink(), s);
m.showGUI();
return m;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
public Map<String, Object> calculateObjects(Map<String, Object> input) {
Map<String, Object> result = new HashMap<String, Object>();
Matrix source = MathUtil.getMatrix(input.get(SOURCE));
Matrix target = Matrix.Factory.zeros(source.getRowCount(), 1);
long cols = source.getColumnCount();
long rows = source.getRowCount();
for (int k = 0; k < rows; k++) {
int tp = 0;
int fn = 0;
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
int count = source.getAsInt(r, c);
boolean expected = r == k;
boolean predicted = c == k;
if (expected && predicted) {
tp += count;
} else if (expected && (!predicted)) {
fn += count;
}
}
}
target.setAsDouble(MathUtil.sensitivity(tp, fn), k, 0);
}
result.put(TARGET, target);
return result;
}
public double getDouble(long... coordinates) {
if (inv == null) {
DenseMatrix A = new MTJDenseDoubleMatrix2D(getSource()).getWrappedObject();
DenseMatrix I = Matrices.identity((int) getSource().getColumnCount());
DenseMatrix AI = I.copy();
inv = new MTJDenseDoubleMatrix2D((DenseMatrix) A.solve(I, AI));
}
return inv.getAsDouble(coordinates);
}
public double getDouble(long... coordinates) {
if (mean == null) {
mean = new Mean(getDimension(), true, getSource()).calcNew();
}
double v = getSource().getAsDouble(coordinates);
if (MathUtil.isNaNOrInfinite(v)) {
switch (getDimension()) {
case ALL:
return mean.getAsDouble(0, 0);
case ROW:
return mean.getAsDouble(0, coordinates[COLUMN]);
case COLUMN:
return mean.getAsDouble(coordinates[ROW], 0);
}
} else {
return v;
}
return 0.0;
}
public double getDouble(long... coordinates) {
if (imputed == null) {
createMatrix();
}
double v = getSource().getAsDouble(coordinates);
if (MathUtil.isNaNOrInfinite(v)) {
return imputed.getAsDouble(coordinates);
} else {
return v;
}
}
public Map<String, Object> calculateObjects(Map<String, Object> input) {
int dimension = defaultDimension;
boolean ignoreNaN = defaultIgnoreNaN;
Map<String, Object> result = new HashMap<String, Object>();
Matrix source = MathUtil.getMatrix(input.get(SOURCE));
Object o2 = input.get(DIMENSION);
if (o2 != null) {
dimension = MathUtil.getInt(o2);
}
Object o3 = input.get(IGNORENAN);
if (o3 != null) {
ignoreNaN = MathUtil.getBoolean(o3);
}
Matrix target = source.mean(Ret.NEW, dimension, ignoreNaN);
result.put(TARGET, target);
return result;
}
private Matrix calculate() {
wordMapping = new DefaultMapMatrix<String, Long>();
Matrix m = getSource();
for (long[] c : m.availableCoordinates()) {
String s = m.getAsString(c);
if (s != null) {
String[] words = s.split("\\s+");
for (String w : words) {
if (w.length() == 0) {
continue;
}
Long i = wordMapping.get(w);
if (i == null) {
wordMapping.put(w, wordMapping.getRowCount());
}
}
}
}
result = new DefaultSparseLongMatrix(m.getRowCount(), wordMapping.getRowCount());
long rowCount = m.getRowCount();
long colCount = m.getColumnCount();
for (long row = 0; row < rowCount; row++) {
for (long col = 0; col < colCount; col++) {
String string = m.getAsString(row, col);
if (string != null && string.length() > 0) {
String[] words = string.split("[\\s]+");
for (String w : words) {
if (w.length() == 0) {
continue;
}
long i = wordMapping.get(w);
int count = result.getAsInt(row, i);
result.setAsInt(++count, row, i);
}
}
}
}
return result;
}
public void asLatex() throws IOException {
final String EOL = System.getProperty("line.separator");
final Writer writer = getWriter();
final Matrix matrix = getMatrix();
final long rowCount = matrix.getRowCount();
final long colCount = matrix.getColumnCount();
writer.write("\\begin{table}[!ht]" + EOL);
writer.write("\\centering" + EOL);
if (matrix.getLabelObject() != null) {
writer.write(
"\\caption{"
+ UJMPFormat.getSingleLineInstance().format(matrix.getLabelObject())
+ "}"
+ EOL);
}
StringBuilder buf = new StringBuilder();
for (long i = matrix.getColumnCount() - 1; i != -1; i--) {
buf.append('c');
}
String alignment = buf.toString();
writer.write("\\begin{tabular}{" + alignment + "}" + EOL);
writer.write("\\toprule" + EOL);
for (int row = 0; row < rowCount; row++) {
for (int col = 0; col < colCount; col++) {
writer.write(UJMPFormat.getSingleLineInstance().format(matrix.getAsObject(row, col)));
if (col < colCount - 1) {
writer.write(" & ");
}
}
writer.write(" \\\\" + EOL);
}
writer.write("\\bottomrule" + EOL);
writer.write("\\end{tabular}" + EOL);
writer.write("\\end{table}" + EOL);
}
private static Matrix replaceInColumn(Matrix original, Matrix firstGuess, long column) {
Matrix x = firstGuess.deleteColumns(Ret.NEW, column);
Matrix y = original.selectColumns(Ret.NEW, column);
List<Long> missingRows = new ArrayList<Long>();
for (long i = y.getRowCount(); --i >= 0; ) {
double v = y.getAsDouble(i, 0);
if (MathUtil.isNaNOrInfinite(v)) {
missingRows.add(i);
}
}
if (missingRows.isEmpty()) {
return y;
}
Matrix xdel = x.deleteRows(Ret.NEW, missingRows);
DenseDoubleMatrix2D bias1 = DenseDoubleMatrix2D.Factory.ones(xdel.getRowCount(), 1);
Matrix xtrain = Matrix.Factory.horCat(xdel, bias1);
Matrix ytrain = y.deleteRows(Ret.NEW, missingRows);
Matrix xinv = xtrain.pinv();
Matrix b = xinv.mtimes(ytrain);
DenseDoubleMatrix2D bias2 = DenseDoubleMatrix2D.Factory.ones(x.getRowCount(), 1);
Matrix yPredicted = Matrix.Factory.horCat(x, bias2).mtimes(b);
// set non-missing values back to original values
for (int row = 0; row < y.getRowCount(); row++) {
double v = y.getAsDouble(row, 0);
if (!Double.isNaN(v)) {
yPredicted.setAsDouble(v, row, 0);
}
}
return yPredicted;
}
public long getLong(long... coordinates) {
if (result == null) {
result = calculate();
}
return result.getAsLong(coordinates);
}
public long[] getSize() {
if (result == null) {
result = calculate();
}
return result.getSize();
}
public void calcscores() {
Matrix mbase, mbaseavg;
Matrix mtemp;
double sco, div;
@SuppressWarnings("unused")
double scodef, divdef;
mbase = clustinit.vtestsm;
mbaseavg = clustinit.avgsm;
scorevar = new double[(int) mbase.getRowCount()];
scorestabdet = new double[(int) mbase.getRowCount()];
scorevardef = new double[(int) mbase.getRowCount()];
scorestabdesc = 0;
scorestabpop = 0;
for (int i = 0; i < 3; i++) {
scorevar[i] = 0;
scorevardef[i] = 0;
}
for (int i = 3; i < mbase.getRowCount(); i++) {
mtemp = mbase.selectRows(Ret.LINK, i);
sco = mtemp.abs(Ret.LINK).getMeanValue();
sco = sco * mbaseavg.selectRows(Ret.LINK, i).getStdValue();
div = mbaseavg.abs(Ret.LINK).selectRows(Ret.LINK, i).getMeanValue();
if (div > 0) {
sco = sco / div;
} else {
sco = 0;
}
scorevar[i] = sco;
}
mbase = clustinit.vtestsmdef;
mbaseavg = clustinit.avgsmdef;
for (int i = 3; i < mbase.getRowCount(); i++) {
mtemp = mbase.selectRows(Ret.LINK, i);
sco = mtemp.abs(Ret.LINK).getMeanValue();
sco = sco * mbaseavg.selectRows(Ret.LINK, i).getStdValue();
div = mbaseavg.abs(Ret.LINK).selectRows(Ret.LINK, i).getMeanValue();
if (div > 0) {
sco = sco / div;
} else {
sco = 0;
}
scorevardef[i] = sco;
}
scorestabdesc = 0;
@SuppressWarnings("unused")
int nbvarninit;
for (int i = 3; i < mbase.getRowCount(); i++) {
scorestabdet[i] = 0;
for (int j = 0; j < mbase.getColumnCount(); j++) {
scorestabdet[i] +=
Math.abs(
clustinit.vtestsm.getAsDouble(i, j)
- clustinit.vtestsm.getAsDouble(i, clustinit.idtickinit))
/ Math.max(
Math.abs(clustinit.vtestsm.getAsDouble(i, j)),
Math.abs(clustinit.vtestsm.getAsDouble(i, clustinit.idtickinit)))
/ mbase.getColumnCount();
}
scorestabdet[i] = 1 - scorestabdet[i];
if (!Double.isNaN(scorestabdet[i])) {
scorestabdesc = scorestabdesc + scorestabdet[i] / (mbase.getRowCount() - 3);
}
}
scorestabpop = 0;
Cluster nc;
for (int j = 0; j < clustHistDef.size(); j++) {
nc = clustHistDef.get(j);
scorestabpop +=
(double) clustinit.getNumberOfCommonComponents(nc)
/ (double) Math.max(clustinit.getComponentIds().size(), nc.getComponentIds().size());
}
scorestabpop = (double) scorestabpop / clustinit.vtestsm.getColumnCount();
long idColumn = mbase.getRowForLabel(Cluster.ID_C_NAME);
long classLabelColumn = mbase.getRowForLabel(Cluster.CLASS_LABEL_C_NAME);
long labelColorColumn = mbase.getRowForLabel("LABEL-COLOR");
for (int i = 3; i < mbase.getRowCount(); i++) {
Pattern p = Pattern.compile("CLASS_LABEL");
Matcher m = p.matcher(mbase.getRowLabel(i));
if ((i == idColumn || i == classLabelColumn || i == labelColorColumn || m.lookingAt())) {
scorevardef[i] = 0;
scorevar[i] = 0;
}
}
}
private void createMatrix() {
try {
ExecutorService executor =
Executors.newFixedThreadPool(UJMPSettings.getInstance().getNumberOfThreads());
Matrix x = getSource();
double valueCount = x.getValueCount();
long missingCount = (long) x.countMissing(Ret.NEW, Matrix.ALL).getEuklideanValue();
double percent = ((int) Math.round((missingCount * 1000.0 / valueCount))) / 10.0;
System.out.println("missing values: " + missingCount + " (" + percent + "%)");
System.out.println("============================================");
if (bestGuess == null) {
bestGuess = getSource().impute(Ret.NEW, ImputationMethod.RowMean);
}
int run = 0;
double d;
do {
System.out.println("Iteration " + run++);
List<Future<Long>> futures = new ArrayList<Future<Long>>();
imputed = Matrix.Factory.zeros(x.getSize());
long t0 = System.currentTimeMillis();
for (long c = 0; c < x.getColumnCount(); c++) {
if (containsMissingValues(c)) {
futures.add(executor.submit(new PredictColumn(c)));
}
}
for (Future<Long> f : futures) {
Long completedCols = f.get();
long elapsedTime = System.currentTimeMillis() - t0;
long remainingCols = x.getColumnCount() - completedCols;
double colsPerMillisecond = (double) (completedCols + 1) / (double) elapsedTime;
long remainingTime = (long) (remainingCols / colsPerMillisecond / 1000.0);
System.out.println(
(completedCols * 1000 / x.getColumnCount() / 10.0)
+ "% completed ("
+ remainingTime
+ " seconds remaining)");
}
Matrix newBestGuess = bestGuess.times(decay).plus(imputed.times(1 - decay));
for (int r = 0; r < getSource().getRowCount(); r++) {
for (int c = 0; c < getSource().getColumnCount(); c++) {
double value = getSource().getAsDouble(r, c);
if (!MathUtil.isNaNOrInfinite(value)) {
newBestGuess.setAsDouble(value, r, c);
}
}
}
d = newBestGuess.euklideanDistanceTo(bestGuess, true) / missingCount;
System.out.println("delta: " + d);
System.out.println("============================================");
bestGuess = newBestGuess;
bestGuess.exportTo().file(tempFile).asDenseCSV();
} while (delta < d);
executor.shutdown();
imputed = bestGuess;
if (imputed.containsMissingValues()) {
throw new RuntimeException("Matrix has still missing values after imputation");
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public Xor(Matrix m1, boolean v2) throws MatrixException {
this(m1, MatrixFactory.fill(v2, m1.getSize()));
}
public Xor(boolean v1, Matrix m2) throws MatrixException {
this(MatrixFactory.fill(v1, m2.getSize()), m2);
}
@Override
protected double sampleAlpha(
Matrix direction, Matrix Aobj, Matrix objConst, double alphaLow, double alphaHigh) {
// Construct distribution over the randomly sampled line given by the direction and the bounds
// [alphaLow, alphaHigh]
Matrix AobjD = Aobj.mtimes(direction);
Matrix AobjP = Aobj.mtimes(currentPt);
OpenLongObjectHashMap alphaMap = new OpenLongObjectHashMap();
long alphaLowBin = Math.round(alphaLow * alphaRoundingScheme);
ObjFunHandle alphaLowHandle = ObjFunHandle.getHandle(alphaLowBin, alphaMap);
for (int i = 0; i < Aobj.getRowCount(); i++) {
double rate = AobjD.getAsDouble(i, 0);
double constant = AobjP.getAsDouble(i, 0) + objConst.getAsDouble(i, 0);
double alpha = (-constant) / rate;
long alphaBin = Math.round(alpha * alphaRoundingScheme);
log.trace("Alpha {}", alpha);
log.trace("Rate {} Constant {}", rate, constant);
if (rate > epsilon && alpha < alphaHigh) {
ObjFunHandle h;
if (alpha <= alphaLow) {
// This evidence is active over the entire interval
h = alphaLowHandle;
} else {
// This evidence will get active once we reach alpha in the interval
h = ObjFunHandle.getHandle(alphaBin, alphaMap);
}
h.increaseBy(rate, constant);
} else if (rate < -epsilon && alpha > alphaLow) {
alphaLowHandle.increaseBy(rate, constant);
if (alpha >= alphaHigh) {
// This evidence is active over the entire interval
} else {
// This evidence will get inactive once we reach alpha in the interval
ObjFunHandle h = ObjFunHandle.getHandle(alphaBin, alphaMap);
h.increaseBy(-rate, -constant);
}
} else { // Considered 0
alphaLowHandle.increaseBy(0.0, constant);
}
}
ObjFunHandle.getHandle(Math.round(alphaHigh * alphaRoundingScheme), alphaMap);
AobjD = null;
AobjP = null;
int length = alphaMap.size();
LongArrayList alphaBinsL = new LongArrayList(length);
ObjectArrayList handlesL = new ObjectArrayList(length);
alphaMap.pairsSortedByKey(alphaBinsL, handlesL);
assert alphaBinsL.size() == length && handlesL.size() == length;
double[] logcumulative = new double[length];
double[] cumRate = new double[length - 1];
double[] cumConst = new double[length - 1];
long[] alphaBins = alphaBinsL.elements();
Object[] handles = handlesL.elements();
log.trace("alphaBins: {}", Arrays.toString(alphaBins));
log.trace("handles: {}", Arrays.toString(handles));
// Now compute the cumulative probability distribution
ObjFunHandle base = (ObjFunHandle) handles[0];
double baserate = base.getRate();
double baseconstant = base.getConstant();
double basealpha = alphaBins[0] / (double) alphaRoundingScheme;
double baselogdivisor = baserate * basealpha + baseconstant;
logcumulative[0] = 0.0;
double rate = 0.0;
double constant = 0.0;
for (int i = 1; i < length; i++) {
ObjFunHandle h = (ObjFunHandle) handles[i - 1];
rate += h.getRate();
cumRate[i - 1] = rate;
constant += h.getConstant();
cumConst[i - 1] = constant;
double aL = alphaBins[i - 1] / (double) alphaRoundingScheme,
aU = alphaBins[i] / (double) alphaRoundingScheme;
double increment;
if (rate < -epsilon || rate > epsilon)
increment =
1.0
/ rate
* (Math.exp(baselogdivisor - rate * aL - constant)
- Math.exp(baselogdivisor - rate * aU - constant));
else increment = (aU - aL) * Math.exp(baselogdivisor - constant);
assert increment > -epsilon : increment;
logcumulative[i] = logcumulative[i - 1] + increment;
}
double total = logcumulative[length - 1];
double randomPt = Math.random() * total;
log.trace("Random Pt {}", randomPt);
log.trace("logcumulative: {}", Arrays.toString(logcumulative));
int findIndex = Arrays.binarySearch(logcumulative, randomPt);
log.trace("Find Index {}", findIndex);
double alphaPt;
if (findIndex >= 0) alphaPt = alphaBins[findIndex] / (double) alphaRoundingScheme;
else {
findIndex = -findIndex - 1;
assert findIndex > 0 : findIndex;
double aL = alphaBins[findIndex - 1] / (double) alphaRoundingScheme;
final double crate = cumRate[findIndex - 1];
final double cconstant = cumConst[findIndex - 1];
log.trace("R {}, c {} aL " + aL, crate, cconstant);
if (crate < -epsilon || crate > epsilon) {
double targetValue =
-randomPt
+ logcumulative[findIndex - 1]
+ 1.0 / crate * Math.exp(-cconstant - crate * aL + baselogdivisor);
log.trace("Target value {}", targetValue);
log.trace("Log: {}, inside log {}", Math.log(crate * targetValue), crate * targetValue);
alphaPt = -1.0 / crate * (Math.log(crate * targetValue) + cconstant - baselogdivisor);
} else {
double kcons = Math.exp(baselogdivisor - cconstant);
alphaPt = randomPt / kcons - logcumulative[findIndex - 1] / kcons + aL;
}
assert !Double.isNaN(alphaPt)
: "C: "
+ cconstant
+ " | R: "
+ crate
+ " Pt: "
+ randomPt
+ " aL: "
+ aL
+ " C0:"
+ logcumulative[findIndex - 1];
}
assert alphaPt >= alphaLow - epsilon && alphaPt <= alphaHigh + epsilon : alphaPt;
alphaPt = Math.max(alphaPt, alphaLow);
alphaPt = Math.min(alphaPt, alphaHigh);
log.trace("Sampled alpha {}", alphaPt);
return alphaPt;
}
public Object call() throws MatrixException {
Matrix m = getMatrixObject().getMatrix().sin(getRet());
m.showGUI();
return m;
}