@Override
protected long[] inferOutputCharacteristics(MemoTable memo) {
long[] ret = null;
Hop input = getInput().get(0);
MatrixCharacteristics mc = memo.getAllInputStats(input);
if (mc.dimsKnown()) {
if (_op == OpOp1.ABS
|| _op == OpOp1.COS
|| _op == OpOp1.SIN
|| _op == OpOp1.TAN
|| _op == OpOp1.ACOS
|| _op == OpOp1.ASIN
|| _op == OpOp1.ATAN
|| _op == OpOp1.SQRT
|| _op == OpOp1.ROUND
|| _op == OpOp1.SPROP
|| _op == OpOp1.SELP) // sparsity preserving
{
ret = new long[] {mc.getRows(), mc.getCols(), mc.getNonZeros()};
} else ret = new long[] {mc.getRows(), mc.getCols(), -1};
}
return ret;
}
@Override
protected long[] inferOutputCharacteristics(MemoTable memo) {
long[] ret = null;
Hop input = getInput().get(0); // original matrix
MatrixCharacteristics mc = memo.getAllInputStats(input);
if (mc != null) {
long lnnz = mc.dimsKnown() ? Math.min(mc.getRows() * mc.getCols(), mc.getNonZeros()) : -1;
// worst-case is input size, but dense
ret = new long[] {mc.getRows(), mc.getCols(), lnnz};
// exploit column/row indexing information
if (_rowLowerEqualsUpper) ret[0] = 1;
if (_colLowerEqualsUpper) ret[1] = 1;
// infer tight block indexing size
Hop rl = getInput().get(1);
Hop ru = getInput().get(2);
Hop cl = getInput().get(3);
Hop cu = getInput().get(4);
if (isBlockIndexingExpression(rl, ru)) ret[0] = getBlockIndexingExpressionSize(rl, ru);
if (isBlockIndexingExpression(cl, cu)) ret[1] = getBlockIndexingExpressionSize(cl, cu);
}
return ret;
}
public void computeMatrixCharacteristics(
MatrixCharacteristics mc1,
MatrixCharacteristics mc2,
MatrixCharacteristics mc3,
MatrixCharacteristics dimOut) {
QuaternaryOperator qop = (QuaternaryOperator) optr;
if (qop.wtype1 != null || qop.wtype4 != null) { // wsloss/wcemm
// output size independent of chain type (scalar)
dimOut.set(1, 1, mc1.getRowsPerBlock(), mc1.getColsPerBlock());
} else if (qop.wtype2 != null || qop.wtype5 != null) { // wsigmoid/wumm
// output size determined by main input
dimOut.set(mc1.getRows(), mc1.getCols(), mc1.getRowsPerBlock(), mc1.getColsPerBlock());
} else if (qop.wtype3 != null) { // wdivmm
// note: cannot directly consume mc2 or mc3 for redwdivmm because rep instruction changed
// the relevant dimensions; as a workaround the original dims are passed via nnz
boolean mapwdivmm = _cacheU && _cacheV;
long rank =
qop.wtype3.isLeft()
? mapwdivmm ? mc3.getCols() : mc3.getNonZeros()
: mapwdivmm ? mc2.getCols() : mc2.getNonZeros();
MatrixCharacteristics mcTmp =
qop.wtype3.computeOutputCharacteristics(mc1.getRows(), mc1.getCols(), rank);
dimOut.set(mcTmp.getRows(), mcTmp.getCols(), mc1.getRowsPerBlock(), mc1.getColsPerBlock());
}
}
@Override
protected long[] inferOutputCharacteristics(MemoTable memo) {
long[] ret = null;
Hop input = getInput().get(0);
MatrixCharacteristics mc = memo.getAllInputStats(input);
if (_direction == Direction.Col && mc.colsKnown()) ret = new long[] {1, mc.getCols(), -1};
else if (_direction == Direction.Row && mc.rowsKnown()) ret = new long[] {mc.getRows(), 1, -1};
return ret;
}
@Override
public void processInstruction(ExecutionContext ec)
throws DMLRuntimeException, DMLUnsupportedOperationException {
SparkExecutionContext sec = (SparkExecutionContext) ec;
MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName());
long rlen = mc.getRows();
int brlen = mc.getRowsPerBlock();
int bclen = mc.getColsPerBlock();
// get input
JavaPairRDD<MatrixIndexes, MatrixBlock> in =
sec.getBinaryBlockRDDHandleForVariable(input1.getName());
// execute unary aggregate (w/ implicit drop correction)
AggregateUnaryOperator auop = (AggregateUnaryOperator) _optr;
JavaPairRDD<MatrixIndexes, MatrixBlock> out =
in.mapToPair(new RDDCumAggFunction(auop, rlen, brlen, bclen));
out = RDDAggregateUtils.mergeByKey(out);
// put output handle in symbol table
sec.setRDDHandleForVariable(output.getName(), out);
sec.addLineageRDD(output.getName(), input1.getName());
}
/**
* @param type
* @param mcBc
* @return
*/
private static boolean requiresFlatMapFunction(CacheType type, MatrixCharacteristics mcBc) {
return (type == CacheType.LEFT && mcBc.getRows() > mcBc.getRowsPerBlock())
|| (type == CacheType.RIGHT && mcBc.getCols() > mcBc.getColsPerBlock());
}
/**
* @param mcIn
* @param type
* @return
*/
private static boolean preservesPartitioning(MatrixCharacteristics mcIn, CacheType type) {
if (type == CacheType.LEFT) return mcIn.dimsKnown() && mcIn.getRows() <= mcIn.getRowsPerBlock();
else // RIGHT
return mcIn.dimsKnown() && mcIn.getCols() <= mcIn.getColsPerBlock();
}
public void setMatrixCharacteristics(MatrixCharacteristics mcIn) {
_mcIn = mcIn;
_lastRowBlockIndex = (long) Math.ceil((double) _mcIn.getRows() / _mcIn.getRowsPerBlock());
}