@Override
public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
throws Exception {
MatrixIndexes ixIn = arg0._1();
MatrixBlock blkIn = arg0._2();
MatrixIndexes ixOut = new MatrixIndexes();
MatrixBlock blkOut = new MatrixBlock();
// process instruction
OperationsOnMatrixValues.performAggregateUnary(
ixIn, blkIn, ixOut, blkOut, ((AggregateUnaryOperator) _op), _brlen, _bclen);
if (((AggregateUnaryOperator) _op).aggOp.correctionExists)
blkOut.dropLastRowsOrColums(((AggregateUnaryOperator) _op).aggOp.correctionLocation);
// cumsum expand partial aggregates
long rlenOut = (long) Math.ceil((double) _rlen / _brlen);
long rixOut = (long) Math.ceil((double) ixIn.getRowIndex() / _brlen);
int rlenBlk = (int) Math.min(rlenOut - (rixOut - 1) * _brlen, _brlen);
int clenBlk = blkOut.getNumColumns();
int posBlk = (int) ((ixIn.getRowIndex() - 1) % _brlen);
MatrixBlock blkOut2 = new MatrixBlock(rlenBlk, clenBlk, false);
blkOut2.copy(posBlk, posBlk, 0, clenBlk - 1, blkOut, true);
ixOut.setIndexes(rixOut, ixOut.getColumnIndex());
// output new tuple
return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut2);
}
@Override
public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
throws Exception {
MatrixIndexes ixIn = arg0._1();
MatrixBlock blkIn = arg0._2();
MatrixIndexes ixOut = new MatrixIndexes();
MatrixBlock blkOut = new MatrixBlock();
if (_type == CacheType.LEFT) {
// get the right hand side matrix
MatrixBlock left = _pbc.getMatrixBlock(1, (int) ixIn.getRowIndex());
// execute matrix-vector mult
OperationsOnMatrixValues.performAggregateBinary(
new MatrixIndexes(1, ixIn.getRowIndex()), left, ixIn, blkIn, ixOut, blkOut, _op);
} else // if( _type == CacheType.RIGHT )
{
// get the right hand side matrix
MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), 1);
// execute matrix-vector mult
OperationsOnMatrixValues.performAggregateBinary(
ixIn, blkIn, new MatrixIndexes(ixIn.getColumnIndex(), 1), right, ixOut, blkOut, _op);
}
// output new tuple
return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut);
}
/**
* @param ix
* @param brlen
* @param bclen
* @param rl
* @param ru
* @param cl
* @param cu
* @return
*/
public static boolean isInBlockRange(
MatrixIndexes ix, int brlen, int bclen, long rl, long ru, long cl, long cu) {
long bRLowerIndex = (ix.getRowIndex() - 1) * brlen + 1;
long bRUpperIndex = ix.getRowIndex() * brlen;
long bCLowerIndex = (ix.getColumnIndex() - 1) * bclen + 1;
long bCUpperIndex = ix.getColumnIndex() * bclen;
if (rl > bRUpperIndex || ru < bRLowerIndex) {
return false;
} else if (cl > bCUpperIndex || cu < bCLowerIndex) {
return false;
} else {
return true;
}
}
@Override
public void processInstruction(
Class<? extends MatrixValue> valueClass,
CachedValueMap cachedValues,
IndexedMatrixValue tempValue,
IndexedMatrixValue zeroInput,
int blockRowFactor,
int blockColFactor)
throws DMLRuntimeException {
ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
if (blkList == null) return;
for (IndexedMatrixValue in1 : blkList) {
if (in1 == null) continue;
MatrixIndexes inix = in1.getIndexes();
MatrixBlock blk = (MatrixBlock) in1.getValue();
long rixOffset = (inix.getRowIndex() - 1) * blockRowFactor;
boolean firstBlk = (inix.getRowIndex() == 1);
boolean lastBlk = (inix.getRowIndex() == _lastRowBlockIndex);
// introduce offsets w/ init value for first row
if (firstBlk) {
IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
((MatrixBlock) out.getValue()).reset(1, blk.getNumColumns());
if (_initValue != 0) {
for (int j = 0; j < blk.getNumColumns(); j++)
((MatrixBlock) out.getValue()).appendValue(0, j, _initValue);
}
out.getIndexes().setIndexes(1, inix.getColumnIndex());
}
// output splitting (shift by one), preaggregated offset used by subsequent block
for (int i = 0; i < blk.getNumRows(); i++)
if (!(lastBlk && i == (blk.getNumRows() - 1))) // ignore last row
{
IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
MatrixBlock tmpBlk = (MatrixBlock) out.getValue();
tmpBlk.reset(1, blk.getNumColumns());
blk.sliceOperations(i, i, 0, blk.getNumColumns() - 1, tmpBlk);
out.getIndexes().setIndexes(rixOffset + i + 2, inix.getColumnIndex());
}
}
}
@Override
public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(
Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception {
ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret =
new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>();
MatrixIndexes ixIn = arg0._1();
MatrixBlock blkIn = arg0._2();
if (_type == CacheType.LEFT) {
// for all matching left-hand-side blocks
int len = _pbc.getNumRowBlocks();
for (int i = 1; i <= len; i++) {
MatrixBlock left = _pbc.getMatrixBlock(i, (int) ixIn.getRowIndex());
MatrixIndexes ixOut = new MatrixIndexes();
MatrixBlock blkOut = new MatrixBlock();
// execute matrix-vector mult
OperationsOnMatrixValues.performAggregateBinary(
new MatrixIndexes(i, ixIn.getRowIndex()), left, ixIn, blkIn, ixOut, blkOut, _op);
ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut));
}
} else // if( _type == CacheType.RIGHT )
{
// for all matching right-hand-side blocks
int len = _pbc.getNumColumnBlocks();
for (int j = 1; j <= len; j++) {
// get the right hand side matrix
MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), j);
MatrixIndexes ixOut = new MatrixIndexes();
MatrixBlock blkOut = new MatrixBlock();
// execute matrix-vector mult
OperationsOnMatrixValues.performAggregateBinary(
ixIn, blkIn, new MatrixIndexes(ixIn.getColumnIndex(), j), right, ixOut, blkOut, _op);
ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut));
}
}
return ret;
}
public static void performAggregateBinary(
MatrixIndexes indexes1,
MatrixValue value1,
MatrixIndexes indexes2,
MatrixValue value2,
MatrixIndexes indexesOut,
MatrixValue valueOut,
AggregateBinaryOperator op)
throws DMLRuntimeException {
// compute output index
indexesOut.setIndexes(indexes1.getRowIndex(), indexes2.getColumnIndex());
// perform on the value
value1.aggregateBinaryOperations(indexes1, value1, indexes2, value2, valueOut, op);
}
@Override
public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(
Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception {
ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret =
new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>();
MatrixIndexes ixIn = arg0._1();
MatrixBlock mb2 = arg0._2();
// get the right hand side matrix
MatrixBlock mb1 = _pmV.getMatrixBlock((int) ixIn.getRowIndex(), 1);
// compute target block indexes
long minPos = UtilFunctions.toLong(mb1.minNonZero());
long maxPos = UtilFunctions.toLong(mb1.max());
long rowIX1 = (minPos - 1) / _brlen + 1;
long rowIX2 = (maxPos - 1) / _brlen + 1;
boolean multipleOuts = (rowIX1 != rowIX2);
if (minPos >= 1) // at least one row selected
{
// output sparsity estimate
double spmb1 = OptimizerUtils.getSparsity(mb1.getNumRows(), 1, mb1.getNonZeros());
long estnnz = (long) (spmb1 * mb2.getNonZeros());
boolean sparse = MatrixBlock.evalSparseFormatInMemory(_brlen, mb2.getNumColumns(), estnnz);
// compute and allocate output blocks
MatrixBlock out1 = new MatrixBlock();
MatrixBlock out2 = multipleOuts ? new MatrixBlock() : null;
out1.reset(_brlen, mb2.getNumColumns(), sparse);
if (out2 != null)
out2.reset(
UtilFunctions.computeBlockSize(_rlen, rowIX2, _brlen), mb2.getNumColumns(), sparse);
// compute core matrix permutation (assumes that out1 has default blocksize,
// hence we do a meta data correction afterwards)
mb1.permutationMatrixMultOperations(mb2, out1, out2);
out1.setNumRows(UtilFunctions.computeBlockSize(_rlen, rowIX1, _brlen));
ret.add(
new Tuple2<MatrixIndexes, MatrixBlock>(
new MatrixIndexes(rowIX1, ixIn.getColumnIndex()), out1));
if (out2 != null)
ret.add(
new Tuple2<MatrixIndexes, MatrixBlock>(
new MatrixIndexes(rowIX2, ixIn.getColumnIndex()), out2));
}
return ret;
}
@Override
public void processInstruction(
Class<? extends MatrixValue> valueClass,
CachedValueMap cachedValues,
IndexedMatrixValue tempValue,
IndexedMatrixValue zeroInput,
int blockRowFactor,
int blockColFactor)
throws DMLUnsupportedOperationException, DMLRuntimeException {
ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
if (blkList != null)
for (IndexedMatrixValue in : blkList) {
if (in == null) continue;
// allocate space for the output value
IndexedMatrixValue out;
if (input == output) out = tempValue;
else out = cachedValues.holdPlace(output, valueClass);
MatrixIndexes inix = in.getIndexes();
// prune unnecessary blocks for trace
if ((((AggregateUnaryOperator) optr).indexFn instanceof ReduceDiag
&& inix.getColumnIndex() != inix.getRowIndex())) {
// do nothing (block not on diagonal); but reset
out.getValue().reset();
} else // general case
{
// process instruction
AggregateUnaryOperator auop = (AggregateUnaryOperator) optr;
OperationsOnMatrixValues.performAggregateUnary(
inix,
in.getValue(),
out.getIndexes(),
out.getValue(),
auop,
blockRowFactor,
blockColFactor);
if (_dropCorr)
((MatrixBlock) out.getValue()).dropLastRowsOrColums(auop.aggOp.correctionLocation);
}
// put the output value in the cache
if (out == tempValue) cachedValues.add(output, out);
}
}
@Override
public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
throws Exception {
MatrixBlock pmV = _pmV.getMatrixBlock(1, 1);
MatrixIndexes ixIn = arg0._1();
MatrixBlock blkIn = arg0._2();
int rowIx = (int) ixIn.getRowIndex();
MatrixIndexes ixOut = new MatrixIndexes(1, 1);
MatrixBlock blkOut = new MatrixBlock();
// execute mapmmchain operation
blkIn.chainMatrixMultOperations(pmV, _pmW.getMatrixBlock(rowIx, 1), blkOut, ChainType.XtwXv);
// output new tuple
return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut);
}
@Override
protected Tuple2<MatrixIndexes, MatrixBlock> computeNext(
Tuple2<MatrixIndexes, MatrixBlock> arg) throws Exception {
MatrixIndexes ixIn = arg._1();
MatrixBlock blkIn = arg._2();
MatrixBlock blkOut = new MatrixBlock();
if (_type == CacheType.LEFT) {
// get the right hand side matrix
MatrixBlock left = _pbc.getMatrixBlock(1, (int) ixIn.getRowIndex());
// execute index preserving matrix multiplication
left.aggregateBinaryOperations(left, blkIn, blkOut, _op);
} else // if( _type == CacheType.RIGHT )
{
// get the right hand side matrix
MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), 1);
// execute index preserving matrix multiplication
blkIn.aggregateBinaryOperations(blkIn, right, blkOut, _op);
}
return new Tuple2<MatrixIndexes, MatrixBlock>(ixIn, blkOut);
}
/**
* @param in
* @param ixrange
* @param brlen
* @param bclen
* @param rlen
* @param clen
* @param outlist
* @throws DMLRuntimeException
*/
public static void performShift(
IndexedMatrixValue in,
IndexRange ixrange,
int brlen,
int bclen,
long rlen,
long clen,
ArrayList<IndexedMatrixValue> outlist)
throws DMLRuntimeException {
MatrixIndexes ix = in.getIndexes();
MatrixBlock mb = (MatrixBlock) in.getValue();
long start_lhs_globalRowIndex = ixrange.rowStart + (ix.getRowIndex() - 1) * brlen;
long start_lhs_globalColIndex = ixrange.colStart + (ix.getColumnIndex() - 1) * bclen;
long end_lhs_globalRowIndex = start_lhs_globalRowIndex + mb.getNumRows() - 1;
long end_lhs_globalColIndex = start_lhs_globalColIndex + mb.getNumColumns() - 1;
long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen);
long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen);
long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen);
long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen);
for (long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
for (long leftColIndex = start_lhs_colIndex;
leftColIndex <= end_lhs_colIndex;
leftColIndex++) {
// Calculate global index of right hand side block
long lhs_rl = Math.max((leftRowIndex - 1) * brlen + 1, start_lhs_globalRowIndex);
long lhs_ru = Math.min(leftRowIndex * brlen, end_lhs_globalRowIndex);
long lhs_cl = Math.max((leftColIndex - 1) * bclen + 1, start_lhs_globalColIndex);
long lhs_cu = Math.min(leftColIndex * bclen, end_lhs_globalColIndex);
int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen);
int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen);
int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen);
int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen);
long rhs_rl = lhs_rl - ixrange.rowStart + 1;
long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
long rhs_cl = lhs_cl - ixrange.colStart + 1;
long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen);
int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen);
int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen);
int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen);
MatrixBlock slicedRHSBlk =
mb.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock());
int lbrlen = UtilFunctions.computeBlockSize(rlen, leftRowIndex, brlen);
int lbclen = UtilFunctions.computeBlockSize(clen, leftColIndex, bclen);
MatrixBlock resultBlock = new MatrixBlock(lbrlen, lbclen, false);
resultBlock =
resultBlock.leftIndexingOperations(
slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, UpdateType.COPY);
outlist.add(
new IndexedMatrixValue(new MatrixIndexes(leftRowIndex, leftColIndex), resultBlock));
}
}
}
/**
* @param path
* @param job
* @param fs
* @param dest
* @param rlen
* @param clen
* @param brlen
* @param bclen
* @throws IOException
*/
@SuppressWarnings("deprecation")
private void readBinaryCellMatrixFromHDFS(
Path path,
JobConf job,
FileSystem fs,
MatrixBlock dest,
long rlen,
long clen,
int brlen,
int bclen)
throws IOException {
boolean sparse = dest.isInSparseFormat();
MatrixIndexes key = new MatrixIndexes();
MatrixCell value = new MatrixCell();
int row = -1;
int col = -1;
try {
for (Path lpath : getSequenceFilePaths(fs, path)) // 1..N files
{
// directly read from sequence files (individual partfiles)
SequenceFile.Reader reader = new SequenceFile.Reader(fs, lpath, job);
try {
if (sparse) {
while (reader.next(key, value)) {
row = (int) key.getRowIndex() - 1;
col = (int) key.getColumnIndex() - 1;
double lvalue = value.getValue();
dest.appendValue(row, col, lvalue);
}
} else {
while (reader.next(key, value)) {
row = (int) key.getRowIndex() - 1;
col = (int) key.getColumnIndex() - 1;
double lvalue = value.getValue();
dest.appendValue(row, col, lvalue);
}
}
} finally {
IOUtilFunctions.closeSilently(reader);
}
}
if (sparse) dest.sortSparseRows();
} catch (Exception ex) {
// post-mortem error handling and bounds checking
if (row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen) {
throw new IOException(
"Matrix cell ["
+ (row + 1)
+ ","
+ (col + 1)
+ "] "
+ "out of overall matrix range [1:"
+ rlen
+ ",1:"
+ clen
+ "].");
} else {
throw new IOException("Unable to read matrix in binary cell format.", ex);
}
}
}
@Override
public void processInstruction(
Class<? extends MatrixValue> valueClass,
CachedValueMap cachedValues,
IndexedMatrixValue tempValue,
IndexedMatrixValue zeroInput,
int blockRowFactor,
int blockColFactor)
throws DMLRuntimeException {
QuaternaryOperator qop = (QuaternaryOperator) optr;
ArrayList<IndexedMatrixValue> blkList = cachedValues.get(_input1);
if (blkList != null)
for (IndexedMatrixValue imv : blkList) {
// Step 1: prepare inputs and output
if (imv == null) continue;
MatrixIndexes inIx = imv.getIndexes();
MatrixValue inVal = imv.getValue();
// allocate space for the output value
IndexedMatrixValue iout = null;
if (output == _input1) iout = tempValue;
else iout = cachedValues.holdPlace(output, valueClass);
MatrixIndexes outIx = iout.getIndexes();
MatrixValue outVal = iout.getValue();
// Step 2: get remaining inputs: Wij, Ui, Vj
MatrixValue Xij = inVal;
// get Wij if existing (null of WeightsType.NONE or WSigmoid any type)
IndexedMatrixValue iWij = (_input4 != -1) ? cachedValues.getFirst(_input4) : null;
MatrixValue Wij = (iWij != null) ? iWij.getValue() : null;
if (null == Wij && qop.hasFourInputs()) {
MatrixBlock mb = new MatrixBlock(1, 1, false);
String[] parts = InstructionUtils.getInstructionParts(instString);
mb.quickSetValue(0, 0, Double.valueOf(parts[4]));
Wij = mb;
}
// get Ui and Vj, potentially through distributed cache
MatrixValue Ui =
(!_cacheU)
? cachedValues.getFirst(_input2).getValue() // U
: MRBaseForCommonInstructions.dcValues
.get(_input2)
.getDataBlock((int) inIx.getRowIndex(), 1)
.getValue();
MatrixValue Vj =
(!_cacheV)
? cachedValues.getFirst(_input3).getValue() // t(V)
: MRBaseForCommonInstructions.dcValues
.get(_input3)
.getDataBlock((int) inIx.getColumnIndex(), 1)
.getValue();
// handle special input case: //V through shuffle -> t(V)
if (Ui.getNumColumns() != Vj.getNumColumns()) {
Vj =
LibMatrixReorg.reorg(
(MatrixBlock) Vj,
new MatrixBlock(Vj.getNumColumns(), Vj.getNumRows(), Vj.isInSparseFormat()),
new ReorgOperator(SwapIndex.getSwapIndexFnObject()));
}
// Step 3: process instruction
Xij.quaternaryOperations(qop, Ui, Vj, Wij, outVal);
// set output indexes
if (qop.wtype1 != null || qop.wtype4 != null) outIx.setIndexes(1, 1); // wsloss
else if (qop.wtype2 != null
|| qop.wtype5 != null
|| qop.wtype3 != null && qop.wtype3.isBasic())
outIx.setIndexes(inIx); // wsigmoid/wdivmm-basic
else { // wdivmm
boolean left = qop.wtype3.isLeft();
outIx.setIndexes(left ? inIx.getColumnIndex() : inIx.getRowIndex(), 1);
}
// put the output value in the cache
if (iout == tempValue) cachedValues.add(output, iout);
}
}