@Override public void processInstruction(ExecutionContext ec) throws DMLRuntimeException, DMLUnsupportedOperationException { SparkExecutionContext sec = (SparkExecutionContext) ec; // get rdd and broadcast inputs JavaPairRDD<MatrixIndexes, MatrixBlock> inX = sec.getBinaryBlockRDDHandleForVariable(_input1.getName()); PartitionedBroadcastMatrix inV = sec.getBroadcastForVariable(_input2.getName()); // execute mapmmchain (guaranteed to have single output block) MatrixBlock out = null; if (_chainType == ChainType.XtXv) { RDDMapMMChainFunction fmmc = new RDDMapMMChainFunction(inV); JavaPairRDD<MatrixIndexes, MatrixBlock> tmp = inX.mapValues(fmmc); out = RDDAggregateUtils.sumStable(tmp); } else { // ChainType.XtwXv PartitionedBroadcastMatrix inW = sec.getBroadcastForVariable(_input3.getName()); RDDMapMMChainFunction2 fmmc = new RDDMapMMChainFunction2(inV, inW); JavaPairRDD<MatrixIndexes, MatrixBlock> tmp = inX.mapToPair(fmmc); out = RDDAggregateUtils.sumStable(tmp); } // put output block into symbol table (no lineage because single block) // this also includes implicit maintenance of matrix characteristics sec.setMatrixOutput(_output.getName(), out); }
@Override public void processInstruction(ExecutionContext ec) throws DMLRuntimeException, DMLUnsupportedOperationException { SparkExecutionContext sec = (SparkExecutionContext) ec; String rddVar = (_type == CacheType.LEFT) ? input2.getName() : input1.getName(); String bcastVar = (_type == CacheType.LEFT) ? input1.getName() : input2.getName(); MatrixCharacteristics mcRdd = sec.getMatrixCharacteristics(rddVar); MatrixCharacteristics mcBc = sec.getMatrixCharacteristics(bcastVar); // get inputs JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(rddVar); PartitionedBroadcastMatrix in2 = sec.getBroadcastForVariable(bcastVar); // empty input block filter if (!_outputEmpty) in1 = in1.filter(new FilterNonEmptyBlocksFunction()); // execute mapmult instruction JavaPairRDD<MatrixIndexes, MatrixBlock> out = null; if (requiresFlatMapFunction(_type, mcBc)) out = in1.flatMapToPair(new RDDFlatMapMMFunction(_type, in2)); else if (preservesPartitioning(mcRdd, _type)) out = in1.mapPartitionsToPair(new RDDMapMMPartitionFunction(_type, in2), true); else out = in1.mapToPair(new RDDMapMMFunction(_type, in2)); // empty output block filter if (!_outputEmpty) out = out.filter(new FilterNonEmptyBlocksFunction()); // perform aggregation if necessary and put output into symbol table if (_aggtype == SparkAggType.SINGLE_BLOCK) { MatrixBlock out2 = RDDAggregateUtils.sumStable(out); // put output block into symbol table (no lineage because single block) // this also includes implicit maintenance of matrix characteristics sec.setMatrixOutput(output.getName(), out2); } else // MULTI_BLOCK or NONE { if (_aggtype == SparkAggType.MULTI_BLOCK) out = RDDAggregateUtils.sumByKeyStable(out); // put output RDD handle into symbol table sec.setRDDHandleForVariable(output.getName(), out); sec.addLineageRDD(output.getName(), rddVar); sec.addLineageBroadcast(output.getName(), bcastVar); // update output statistics if not inferred updateBinaryMMOutputMatrixCharacteristics(sec, true); } }
@Override public void processInstruction(ExecutionContext ec) throws DMLRuntimeException, DMLUnsupportedOperationException { SparkExecutionContext sec = (SparkExecutionContext) ec; String rddVar = (_type == CacheType.LEFT) ? input2.getName() : input1.getName(); String bcastVar = (_type == CacheType.LEFT) ? input1.getName() : input2.getName(); MatrixCharacteristics mc = sec.getMatrixCharacteristics(output.getName()); long rlen = sec.getScalarInput(_nrow.getName(), _nrow.getValueType(), _nrow.isLiteral()).getLongValue(); // get inputs JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(rddVar); PartitionedBroadcastMatrix in2 = sec.getBroadcastForVariable(bcastVar); // execute pmm instruction JavaPairRDD<MatrixIndexes, MatrixBlock> out = in1.flatMapToPair(new RDDPMMFunction(_type, in2, rlen, mc.getRowsPerBlock())); out = RDDAggregateUtils.sumByKeyStable(out); // put output RDD handle into symbol table sec.setRDDHandleForVariable(output.getName(), out); sec.addLineageRDD(output.getName(), rddVar); sec.addLineageBroadcast(output.getName(), bcastVar); // update output statistics if not inferred updateBinaryMMOutputMatrixCharacteristics(sec, false); }
@Override public void processInstruction(ExecutionContext ec) throws DMLRuntimeException, DMLUnsupportedOperationException { SparkExecutionContext sec = (SparkExecutionContext) ec; MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName()); long rlen = mc.getRows(); int brlen = mc.getRowsPerBlock(); int bclen = mc.getColsPerBlock(); // get input JavaPairRDD<MatrixIndexes, MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(input1.getName()); // execute unary aggregate (w/ implicit drop correction) AggregateUnaryOperator auop = (AggregateUnaryOperator) _optr; JavaPairRDD<MatrixIndexes, MatrixBlock> out = in.mapToPair(new RDDCumAggFunction(auop, rlen, brlen, bclen)); out = RDDAggregateUtils.mergeByKey(out); // put output handle in symbol table sec.setRDDHandleForVariable(output.getName(), out); sec.addLineageRDD(output.getName(), input1.getName()); }