@Override protected ExecType optFindExecType() throws HopsException { checkAndSetForcedPlatform(); ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR; if (_etypeForced != null) { _etype = _etypeForced; } else { if (OptimizerUtils.isMemoryBasedOptLevel()) { _etype = findExecTypeByMemEstimate(); } else if (getInput().get(0).areDimsBelowThreshold()) { _etype = ExecType.CP; } else { _etype = REMOTE; } // check for valid CP dimensions and matrix size checkAndSetInvalidCPDimsAndSize(); } // mark for recompile (forever) if (OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype == REMOTE) setRequiresRecompile(); return _etype; }
@Override protected ExecType optFindExecType() throws HopsException { checkAndSetForcedPlatform(); ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR; if (_etypeForced != null) { _etype = _etypeForced; } else { if (OptimizerUtils.isMemoryBasedOptLevel()) { _etype = findExecTypeByMemEstimate(); } // Choose CP, if the input dimensions are below threshold or if the input is a vector // Also, matrix inverse is currently implemented only in CP (through commons math) else if (getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector() || isInMemoryOperation()) { _etype = ExecType.CP; } else { _etype = REMOTE; } // check for valid CP dimensions and matrix size checkAndSetInvalidCPDimsAndSize(); } // mark for recompile (forever) if (OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype == REMOTE) setRequiresRecompile(); if (_op == OpOp1.PRINT || _op == OpOp1.STOP || _op == OpOp1.INVERSE) _etype = ExecType.CP; return _etype; }
@Override protected ExecType optFindExecType() throws HopsException { checkAndSetForcedPlatform(); ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR; if (_etypeForced != null) { _etype = _etypeForced; } else { if (OptimizerUtils.isMemoryBasedOptLevel()) { _etype = findExecTypeByMemEstimate(); } // Choose CP, if the input dimensions are below threshold or if the input is a vector // Also, matrix inverse is currently implemented only in CP (through commons math) else if (getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector() || isInMemoryOperation()) { _etype = ExecType.CP; } else { _etype = REMOTE; } // check for valid CP dimensions and matrix size checkAndSetInvalidCPDimsAndSize(); } // spark-specific decision refinement (execute unary w/ spark input and // single parent also in spark because it's likely cheap and reduces intermediates) if (_etype == ExecType.CP && _etypeForced != ExecType.CP && getInput().get(0).optFindExecType() == ExecType.SPARK && getDataType().isMatrix() && !isCumulativeUnaryOperation() && !isCastUnaryOperation() && _op != OpOp1.MEDIAN && _op != OpOp1.IQM && !(getInput().get(0) instanceof DataOp) // input is not checkpoint && getInput().get(0).getParent().size() == 1) // unary is only parent { // pull unary operation into spark _etype = ExecType.SPARK; } // mark for recompile (forever) if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype == REMOTE) setRequiresRecompile(); // ensure cp exec type for single-node operations if (_op == OpOp1.PRINT || _op == OpOp1.STOP || _op == OpOp1.INVERSE || _op == OpOp1.EIGEN || _op == OpOp1.CHOLESKY) { _etype = ExecType.CP; } return _etype; }
@Override protected ExecType optFindExecType() throws HopsException { checkAndSetForcedPlatform(); ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR; // forced / memory-based / threshold-based decision if (_etypeForced != null) { _etype = _etypeForced; } else { if (OptimizerUtils.isMemoryBasedOptLevel()) { _etype = findExecTypeByMemEstimate(); } // Choose CP, if the input dimensions are below threshold or if the input is a vector else if (getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector()) { _etype = ExecType.CP; } else { _etype = REMOTE; } // check for valid CP dimensions and matrix size checkAndSetInvalidCPDimsAndSize(); } // spark-specific decision refinement (execute unary aggregate w/ spark input and // single parent also in spark because it's likely cheap and reduces data transfer) if (_etype == ExecType.CP && _etypeForced != ExecType.CP && !(getInput().get(0) instanceof DataOp) // input is not checkpoint && getInput().get(0).getParent().size() == 1 // uagg is only parent && getInput().get(0).optFindExecType() == ExecType.SPARK) { // pull unary aggregate into spark _etype = ExecType.SPARK; } // mark for recompile (forever) if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype == REMOTE) { setRequiresRecompile(); } return _etype; }