@Override
protected ExecType optFindExecType() throws HopsException {
checkAndSetForcedPlatform();
ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
if (_etypeForced != null) {
_etype = _etypeForced;
} else {
if (OptimizerUtils.isMemoryBasedOptLevel()) {
_etype = findExecTypeByMemEstimate();
}
// Choose CP, if the input dimensions are below threshold or if the input is a vector
// Also, matrix inverse is currently implemented only in CP (through commons math)
else if (getInput().get(0).areDimsBelowThreshold()
|| getInput().get(0).isVector()
|| isInMemoryOperation()) {
_etype = ExecType.CP;
} else {
_etype = REMOTE;
}
// check for valid CP dimensions and matrix size
checkAndSetInvalidCPDimsAndSize();
}
// spark-specific decision refinement (execute unary w/ spark input and
// single parent also in spark because it's likely cheap and reduces intermediates)
if (_etype == ExecType.CP
&& _etypeForced != ExecType.CP
&& getInput().get(0).optFindExecType() == ExecType.SPARK
&& getDataType().isMatrix()
&& !isCumulativeUnaryOperation()
&& !isCastUnaryOperation()
&& _op != OpOp1.MEDIAN
&& _op != OpOp1.IQM
&& !(getInput().get(0) instanceof DataOp) // input is not checkpoint
&& getInput().get(0).getParent().size() == 1) // unary is only parent
{
// pull unary operation into spark
_etype = ExecType.SPARK;
}
// mark for recompile (forever)
if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype == REMOTE)
setRequiresRecompile();
// ensure cp exec type for single-node operations
if (_op == OpOp1.PRINT
|| _op == OpOp1.STOP
|| _op == OpOp1.INVERSE
|| _op == OpOp1.EIGEN
|| _op == OpOp1.CHOLESKY) {
_etype = ExecType.CP;
}
return _etype;
}
@Override
protected ExecType optFindExecType() throws HopsException {
checkAndSetForcedPlatform();
ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
// forced / memory-based / threshold-based decision
if (_etypeForced != null) {
_etype = _etypeForced;
} else {
if (OptimizerUtils.isMemoryBasedOptLevel()) {
_etype = findExecTypeByMemEstimate();
}
// Choose CP, if the input dimensions are below threshold or if the input is a vector
else if (getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector()) {
_etype = ExecType.CP;
} else {
_etype = REMOTE;
}
// check for valid CP dimensions and matrix size
checkAndSetInvalidCPDimsAndSize();
}
// spark-specific decision refinement (execute unary aggregate w/ spark input and
// single parent also in spark because it's likely cheap and reduces data transfer)
if (_etype == ExecType.CP
&& _etypeForced != ExecType.CP
&& !(getInput().get(0) instanceof DataOp) // input is not checkpoint
&& getInput().get(0).getParent().size() == 1 // uagg is only parent
&& getInput().get(0).optFindExecType() == ExecType.SPARK) {
// pull unary aggregate into spark
_etype = ExecType.SPARK;
}
// mark for recompile (forever)
if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype == REMOTE) {
setRequiresRecompile();
}
return _etype;
}
