@Override
public boolean compare(Hop that) {
if (!(that instanceof IndexingOp) || getInput().size() != that.getInput().size()) {
return false;
}
return (getInput().get(0) == that.getInput().get(0)
&& getInput().get(1) == that.getInput().get(1)
&& getInput().get(2) == that.getInput().get(2)
&& getInput().get(3) == that.getInput().get(3)
&& getInput().get(4) == that.getInput().get(4));
}
private Lop constructLopsTernaryAggregateRewrite(ExecType et)
throws HopsException, LopsException {
Hop input1 = getInput().get(0);
Hop input11 = input1.getInput().get(0);
Hop input12 = input1.getInput().get(1);
Lop ret = null;
Lop in1 = null;
Lop in2 = null;
Lop in3 = null;
if (input11 instanceof BinaryOp && ((BinaryOp) input11).getOp() == OpOp2.MULT) {
in1 = input11.getInput().get(0).constructLops();
in2 = input11.getInput().get(1).constructLops();
in3 = input12.constructLops();
} else if (input12 instanceof BinaryOp && ((BinaryOp) input12).getOp() == OpOp2.MULT) {
in1 = input11.constructLops();
in2 = input12.getInput().get(0).constructLops();
in3 = input12.getInput().get(1).constructLops();
} else {
in1 = input11.constructLops();
in2 = input12.constructLops();
in3 = new LiteralOp(1).constructLops();
}
// create new ternary aggregate operator
int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
// The execution type of a unary aggregate instruction should depend on the execution type of
// inputs to avoid OOM
// Since we only support matrix-vector and not vector-matrix, checking the execution type of
// input1 should suffice.
ExecType et_input = input1.optFindExecType();
ret =
new TernaryAggregate(
in1,
in2,
in3,
Aggregate.OperationTypes.KahanSum,
Binary.OperationTypes.MULTIPLY,
DataType.SCALAR,
ValueType.DOUBLE,
et_input,
k);
return ret;
}
private boolean isTernaryAggregateRewriteApplicable() throws HopsException {
boolean ret = false;
// currently we support only sum over binary multiply but potentially
// it can be generalized to any RC aggregate over two common binary operations
if (OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES
&& _direction == Direction.RowCol
&& _op == AggOp.SUM) {
Hop input1 = getInput().get(0);
if (input1.getParent().size() == 1
&& // sum single consumer
input1 instanceof BinaryOp
&& ((BinaryOp) input1).getOp() == OpOp2.MULT
// As unary agg instruction is not implemented in MR and since MR is in maintenance mode,
// postponed it.
&& input1.optFindExecType() != ExecType.MR) {
Hop input11 = input1.getInput().get(0);
Hop input12 = input1.getInput().get(1);
if (input11 instanceof BinaryOp && ((BinaryOp) input11).getOp() == OpOp2.MULT) {
// ternary, arbitrary matrices but no mv/outer operations.
ret =
HopRewriteUtils.isEqualSize(input11.getInput().get(0), input1)
&& HopRewriteUtils.isEqualSize(input11.getInput().get(1), input1)
&& HopRewriteUtils.isEqualSize(input12, input1);
} else if (input12 instanceof BinaryOp && ((BinaryOp) input12).getOp() == OpOp2.MULT) {
// ternary, arbitrary matrices but no mv/outer operations.
ret =
HopRewriteUtils.isEqualSize(input12.getInput().get(0), input1)
&& HopRewriteUtils.isEqualSize(input12.getInput().get(1), input1)
&& HopRewriteUtils.isEqualSize(input11, input1);
} else {
// binary, arbitrary matrices but no mv/outer operations.
ret = HopRewriteUtils.isEqualSize(input11, input12);
}
}
}
return ret;
}
private boolean isUnaryAggregateOuterRewriteApplicable() {
boolean ret = false;
Hop input = getInput().get(0);
if (input instanceof BinaryOp && ((BinaryOp) input).isOuterVectorOperator()) {
// for special cases, we need to hold the broadcast twice in order to allow for
// an efficient binary search over a plain java array
double factor =
(isCompareOperator(((BinaryOp) input).getOp())
&& (_direction == Direction.Row
|| _direction == Direction.Col
|| _direction == Direction.RowCol)
&& (_op == AggOp.SUM))
? 2.0
: 1.0;
factor +=
(isCompareOperator(((BinaryOp) input).getOp())
&& (_direction == Direction.Row || _direction == Direction.Col)
&& (_op == AggOp.MAXINDEX || _op == AggOp.MININDEX))
? 1.0
: 0.0;
// note: memory constraint only needs to take the rhs into account because the output
// is guaranteed to be an aggregate of <=16KB
Hop right = input.getInput().get(1);
if ((right.dimsKnown()
&& factor * OptimizerUtils.estimateSize(right.getDim1(), right.getDim2())
< OptimizerUtils.getRemoteMemBudgetMap(true)) // dims known and estimate fits
|| (!right.dimsKnown()
&& factor * right.getOutputMemEstimate()
< OptimizerUtils.getRemoteMemBudgetMap(
true))) // dims unknown but worst-case estimate fits
{
ret = true;
}
}
return ret;
}
/**
* Indicates if the lbound:rbound expressions is of the form "(c * (i - 1) + 1) : (c * i)", where
* we could use c as a tight size estimate.
*
* @param lbound
* @param ubound
* @return
*/
private boolean isBlockIndexingExpression(Hop lbound, Hop ubound) {
boolean ret = false;
LiteralOp constant = null;
DataOp var = null;
// handle lower bound
if (lbound instanceof BinaryOp
&& ((BinaryOp) lbound).getOp() == OpOp2.PLUS
&& lbound.getInput().get(1) instanceof LiteralOp
&& HopRewriteUtils.getDoubleValueSafe((LiteralOp) lbound.getInput().get(1)) == 1
&& lbound.getInput().get(0) instanceof BinaryOp) {
BinaryOp lmult = (BinaryOp) lbound.getInput().get(0);
if (lmult.getOp() == OpOp2.MULT
&& lmult.getInput().get(0) instanceof LiteralOp
&& lmult.getInput().get(1) instanceof BinaryOp) {
BinaryOp lminus = (BinaryOp) lmult.getInput().get(1);
if (lminus.getOp() == OpOp2.MINUS
&& lminus.getInput().get(1) instanceof LiteralOp
&& HopRewriteUtils.getDoubleValueSafe((LiteralOp) lminus.getInput().get(1)) == 1
&& lminus.getInput().get(0) instanceof DataOp) {
constant = (LiteralOp) lmult.getInput().get(0);
var = (DataOp) lminus.getInput().get(0);
}
}
}
// handle upper bound
if (var != null
&& constant != null
&& ubound instanceof BinaryOp
&& ubound.getInput().get(0) instanceof LiteralOp
&& ubound.getInput().get(1) instanceof DataOp
&& ubound.getInput().get(1).getName().equals(var.getName())) {
LiteralOp constant2 = (LiteralOp) ubound.getInput().get(0);
ret =
(HopRewriteUtils.getDoubleValueSafe(constant)
== HopRewriteUtils.getDoubleValueSafe(constant2));
}
return ret;
}
/**
* This will check if there is sufficient memory locally (twice the size of second matrix, for
* original and sort data), and remotely (size of second matrix (sorted data)).
*
* @return true if sufficient memory
*/
private boolean isUnaryAggregateOuterSPRewriteApplicable() {
boolean ret = false;
Hop input = getInput().get(0);
if (input instanceof BinaryOp && ((BinaryOp) input).isOuterVectorOperator()) {
// note: both cases (partitioned matrix, and sorted double array), require to
// fit the broadcast twice into the local memory budget. Also, the memory
// constraint only needs to take the rhs into account because the output is
// guaranteed to be an aggregate of <=16KB
Hop right = input.getInput().get(1);
double size =
right.dimsKnown()
? OptimizerUtils.estimateSize(right.getDim1(), right.getDim2())
: // dims known and estimate fits
right.getOutputMemEstimate(); // dims unknown but worst-case estimate fits
if (_op == AggOp.MAXINDEX || _op == AggOp.MININDEX) {
double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
// basic requirement: the broadcast needs to to fit twice in the remote broadcast memory
// and local memory budget because we have to create a partitioned broadcast
// memory and hand it over to the spark context as in-memory object
ret = (2 * size < memBudgetExec && 2 * size < memBudgetLocal);
} else {
if (OptimizerUtils.checkSparkBroadcastMemoryBudget(size)) {
ret = true;
}
}
}
return ret;
}
/**
* @param lbound
* @param ubound
* @return
*/
private long getBlockIndexingExpressionSize(Hop lbound, Hop ubound) {
// NOTE: ensure consistency with isBlockIndexingExpression
LiteralOp c = (LiteralOp) ubound.getInput().get(0); // (c*i)
return HopRewriteUtils.getIntValueSafe(c);
}