@Override
public void processInstruction(ExecutionContext ec)
throws DMLRuntimeException, DMLUnsupportedOperationException {
SparkExecutionContext sec = (SparkExecutionContext) ec;
// get rdd and broadcast inputs
JavaPairRDD<MatrixIndexes, MatrixBlock> inX =
sec.getBinaryBlockRDDHandleForVariable(_input1.getName());
PartitionedBroadcastMatrix inV = sec.getBroadcastForVariable(_input2.getName());
// execute mapmmchain (guaranteed to have single output block)
MatrixBlock out = null;
if (_chainType == ChainType.XtXv) {
RDDMapMMChainFunction fmmc = new RDDMapMMChainFunction(inV);
JavaPairRDD<MatrixIndexes, MatrixBlock> tmp = inX.mapValues(fmmc);
out = RDDAggregateUtils.sumStable(tmp);
} else { // ChainType.XtwXv
PartitionedBroadcastMatrix inW = sec.getBroadcastForVariable(_input3.getName());
RDDMapMMChainFunction2 fmmc = new RDDMapMMChainFunction2(inV, inW);
JavaPairRDD<MatrixIndexes, MatrixBlock> tmp = inX.mapToPair(fmmc);
out = RDDAggregateUtils.sumStable(tmp);
}
// put output block into symbol table (no lineage because single block)
// this also includes implicit maintenance of matrix characteristics
sec.setMatrixOutput(_output.getName(), out);
}
@Override
public void processInstruction(ExecutionContext ec)
throws DMLRuntimeException, DMLUnsupportedOperationException {
SparkExecutionContext sec = (SparkExecutionContext) ec;
String rddVar = (_type == CacheType.LEFT) ? input2.getName() : input1.getName();
String bcastVar = (_type == CacheType.LEFT) ? input1.getName() : input2.getName();
MatrixCharacteristics mcRdd = sec.getMatrixCharacteristics(rddVar);
MatrixCharacteristics mcBc = sec.getMatrixCharacteristics(bcastVar);
// get inputs
JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(rddVar);
PartitionedBroadcastMatrix in2 = sec.getBroadcastForVariable(bcastVar);
// empty input block filter
if (!_outputEmpty) in1 = in1.filter(new FilterNonEmptyBlocksFunction());
// execute mapmult instruction
JavaPairRDD<MatrixIndexes, MatrixBlock> out = null;
if (requiresFlatMapFunction(_type, mcBc))
out = in1.flatMapToPair(new RDDFlatMapMMFunction(_type, in2));
else if (preservesPartitioning(mcRdd, _type))
out = in1.mapPartitionsToPair(new RDDMapMMPartitionFunction(_type, in2), true);
else out = in1.mapToPair(new RDDMapMMFunction(_type, in2));
// empty output block filter
if (!_outputEmpty) out = out.filter(new FilterNonEmptyBlocksFunction());
// perform aggregation if necessary and put output into symbol table
if (_aggtype == SparkAggType.SINGLE_BLOCK) {
MatrixBlock out2 = RDDAggregateUtils.sumStable(out);
// put output block into symbol table (no lineage because single block)
// this also includes implicit maintenance of matrix characteristics
sec.setMatrixOutput(output.getName(), out2);
} else // MULTI_BLOCK or NONE
{
if (_aggtype == SparkAggType.MULTI_BLOCK) out = RDDAggregateUtils.sumByKeyStable(out);
// put output RDD handle into symbol table
sec.setRDDHandleForVariable(output.getName(), out);
sec.addLineageRDD(output.getName(), rddVar);
sec.addLineageBroadcast(output.getName(), bcastVar);
// update output statistics if not inferred
updateBinaryMMOutputMatrixCharacteristics(sec, true);
}
}
@Override
public void processInstruction(ExecutionContext ec)
throws DMLRuntimeException, DMLUnsupportedOperationException {
SparkExecutionContext sec = (SparkExecutionContext) ec;
String rddVar = (_type == CacheType.LEFT) ? input2.getName() : input1.getName();
String bcastVar = (_type == CacheType.LEFT) ? input1.getName() : input2.getName();
MatrixCharacteristics mc = sec.getMatrixCharacteristics(output.getName());
long rlen =
sec.getScalarInput(_nrow.getName(), _nrow.getValueType(), _nrow.isLiteral()).getLongValue();
// get inputs
JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(rddVar);
PartitionedBroadcastMatrix in2 = sec.getBroadcastForVariable(bcastVar);
// execute pmm instruction
JavaPairRDD<MatrixIndexes, MatrixBlock> out =
in1.flatMapToPair(new RDDPMMFunction(_type, in2, rlen, mc.getRowsPerBlock()));
out = RDDAggregateUtils.sumByKeyStable(out);
// put output RDD handle into symbol table
sec.setRDDHandleForVariable(output.getName(), out);
sec.addLineageRDD(output.getName(), rddVar);
sec.addLineageBroadcast(output.getName(), bcastVar);
// update output statistics if not inferred
updateBinaryMMOutputMatrixCharacteristics(sec, false);
}
@Override
public void processInstruction(ExecutionContext ec)
throws DMLRuntimeException, DMLUnsupportedOperationException {
SparkExecutionContext sec = (SparkExecutionContext) ec;
MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName());
long rlen = mc.getRows();
int brlen = mc.getRowsPerBlock();
int bclen = mc.getColsPerBlock();
// get input
JavaPairRDD<MatrixIndexes, MatrixBlock> in =
sec.getBinaryBlockRDDHandleForVariable(input1.getName());
// execute unary aggregate (w/ implicit drop correction)
AggregateUnaryOperator auop = (AggregateUnaryOperator) _optr;
JavaPairRDD<MatrixIndexes, MatrixBlock> out =
in.mapToPair(new RDDCumAggFunction(auop, rlen, brlen, bclen));
out = RDDAggregateUtils.mergeByKey(out);
// put output handle in symbol table
sec.setRDDHandleForVariable(output.getName(), out);
sec.addLineageRDD(output.getName(), input1.getName());
}
