/**
* This will check if there is sufficient memory locally (twice the size of second matrix, for
* original and sort data), and remotely (size of second matrix (sorted data)).
*
* @return true if sufficient memory
*/
private boolean isUnaryAggregateOuterSPRewriteApplicable() {
boolean ret = false;
Hop input = getInput().get(0);
if (input instanceof BinaryOp && ((BinaryOp) input).isOuterVectorOperator()) {
// note: both cases (partitioned matrix, and sorted double array), require to
// fit the broadcast twice into the local memory budget. Also, the memory
// constraint only needs to take the rhs into account because the output is
// guaranteed to be an aggregate of <=16KB
Hop right = input.getInput().get(1);
double size =
right.dimsKnown()
? OptimizerUtils.estimateSize(right.getDim1(), right.getDim2())
: // dims known and estimate fits
right.getOutputMemEstimate(); // dims unknown but worst-case estimate fits
if (_op == AggOp.MAXINDEX || _op == AggOp.MININDEX) {
double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget();
double memBudgetLocal = OptimizerUtils.getLocalMemBudget();
// basic requirement: the broadcast needs to to fit twice in the remote broadcast memory
// and local memory budget because we have to create a partitioned broadcast
// memory and hand it over to the spark context as in-memory object
ret = (2 * size < memBudgetExec && 2 * size < memBudgetLocal);
} else {
if (OptimizerUtils.checkSparkBroadcastMemoryBudget(size)) {
ret = true;
}
}
}
return ret;
}
/**
* @return
* @throws HopsException
* @throws LopsException
*/
private Lop constructLopsSparkCumulativeUnary() throws HopsException, LopsException {
Hop input = getInput().get(0);
long rlen = input.getDim1();
long clen = input.getDim2();
long brlen = input.getRowsInBlock();
long bclen = input.getColsInBlock();
boolean force = !dimsKnown() || _etypeForced == ExecType.SPARK;
OperationTypes aggtype = getCumulativeAggType();
Lop X = input.constructLops();
Lop TEMP = X;
ArrayList<Lop> DATA = new ArrayList<Lop>();
int level = 0;
// recursive preaggregation until aggregates fit into CP memory budget
while (((2 * OptimizerUtils.estimateSize(TEMP.getOutputParameters().getNumRows(), clen)
+ OptimizerUtils.estimateSize(1, clen))
> OptimizerUtils.getLocalMemBudget()
&& TEMP.getOutputParameters().getNumRows() > 1)
|| force) {
DATA.add(TEMP);
// preaggregation per block (for spark, the CumulativePartialAggregate subsumes both
// the preaggregation and subsequent block aggregation)
long rlenAgg = (long) Math.ceil((double) TEMP.getOutputParameters().getNumRows() / brlen);
Lop preagg =
new CumulativePartialAggregate(
TEMP, DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.SPARK);
preagg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1);
setLineNumbers(preagg);
TEMP = preagg;
level++;
force = false; // in case of unknowns, generate one level
}
// in-memory cum sum (of partial aggregates)
if (TEMP.getOutputParameters().getNumRows() != 1) {
int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
Unary unary1 =
new Unary(
TEMP, HopsOpOp1LopsU.get(_op), DataType.MATRIX, ValueType.DOUBLE, ExecType.CP, k);
unary1
.getOutputParameters()
.setDimensions(TEMP.getOutputParameters().getNumRows(), clen, brlen, bclen, -1);
setLineNumbers(unary1);
TEMP = unary1;
}
// split, group and mr cumsum
while (level-- > 0) {
// (for spark, the CumulativeOffsetBinary subsumes both the split aggregate and
// the subsequent offset binary apply of split aggregates against the original data)
double initValue = getCumulativeInitValue();
CumulativeOffsetBinary binary =
new CumulativeOffsetBinary(
DATA.get(level),
TEMP,
DataType.MATRIX,
ValueType.DOUBLE,
initValue,
aggtype,
ExecType.SPARK);
binary.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
setLineNumbers(binary);
TEMP = binary;
}
return TEMP;
}
/**
* MR Cumsum is currently based on a multipass algorithm of (1) preaggregation and (2) subsequent
* offsetting. Note that we currently support one robust physical operator but many alternative
* realizations are possible for specific scenarios (e.g., when the preaggregated intermediate fit
* into the map task memory budget) or by creating custom job types.
*
* @return
* @throws HopsException
* @throws LopsException
*/
private Lop constructLopsMRCumulativeUnary() throws HopsException, LopsException {
Hop input = getInput().get(0);
long rlen = input.getDim1();
long clen = input.getDim2();
long brlen = input.getRowsInBlock();
long bclen = input.getColsInBlock();
boolean force = !dimsKnown() || _etypeForced == ExecType.MR;
OperationTypes aggtype = getCumulativeAggType();
Lop X = input.constructLops();
Lop TEMP = X;
ArrayList<Lop> DATA = new ArrayList<Lop>();
int level = 0;
// recursive preaggregation until aggregates fit into CP memory budget
while (((2 * OptimizerUtils.estimateSize(TEMP.getOutputParameters().getNumRows(), clen)
+ OptimizerUtils.estimateSize(1, clen))
> OptimizerUtils.getLocalMemBudget()
&& TEMP.getOutputParameters().getNumRows() > 1)
|| force) {
DATA.add(TEMP);
// preaggregation per block
long rlenAgg = (long) Math.ceil((double) TEMP.getOutputParameters().getNumRows() / brlen);
Lop preagg =
new CumulativePartialAggregate(
TEMP, DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.MR);
preagg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1);
setLineNumbers(preagg);
Group group = new Group(preagg, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
group.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1);
setLineNumbers(group);
Aggregate agg =
new Aggregate(
group, HopsAgg2Lops.get(AggOp.SUM), getDataType(), getValueType(), ExecType.MR);
agg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1);
agg.setupCorrectionLocation(
CorrectionLocationType
.NONE); // aggregation uses kahanSum but the inputs do not have correction values
setLineNumbers(agg);
TEMP = agg;
level++;
force = false; // in case of unknowns, generate one level
}
// in-memory cum sum (of partial aggregates)
if (TEMP.getOutputParameters().getNumRows() != 1) {
int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
Unary unary1 =
new Unary(
TEMP, HopsOpOp1LopsU.get(_op), DataType.MATRIX, ValueType.DOUBLE, ExecType.CP, k);
unary1
.getOutputParameters()
.setDimensions(TEMP.getOutputParameters().getNumRows(), clen, brlen, bclen, -1);
setLineNumbers(unary1);
TEMP = unary1;
}
// split, group and mr cumsum
while (level-- > 0) {
double init = getCumulativeInitValue();
CumulativeSplitAggregate split =
new CumulativeSplitAggregate(TEMP, DataType.MATRIX, ValueType.DOUBLE, init);
split.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
setLineNumbers(split);
Group group1 =
new Group(DATA.get(level), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
group1.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
setLineNumbers(group1);
Group group2 = new Group(split, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
group2.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
setLineNumbers(group2);
CumulativeOffsetBinary binary =
new CumulativeOffsetBinary(
group1, group2, DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.MR);
binary.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1);
setLineNumbers(binary);
TEMP = binary;
}
return TEMP;
}
