private boolean isUnaryAggregateOuterRewriteApplicable() { boolean ret = false; Hop input = getInput().get(0); if (input instanceof BinaryOp && ((BinaryOp) input).isOuterVectorOperator()) { // for special cases, we need to hold the broadcast twice in order to allow for // an efficient binary search over a plain java array double factor = (isCompareOperator(((BinaryOp) input).getOp()) && (_direction == Direction.Row || _direction == Direction.Col || _direction == Direction.RowCol) && (_op == AggOp.SUM)) ? 2.0 : 1.0; factor += (isCompareOperator(((BinaryOp) input).getOp()) && (_direction == Direction.Row || _direction == Direction.Col) && (_op == AggOp.MAXINDEX || _op == AggOp.MININDEX)) ? 1.0 : 0.0; // note: memory constraint only needs to take the rhs into account because the output // is guaranteed to be an aggregate of <=16KB Hop right = input.getInput().get(1); if ((right.dimsKnown() && factor * OptimizerUtils.estimateSize(right.getDim1(), right.getDim2()) < OptimizerUtils.getRemoteMemBudgetMap(true)) // dims known and estimate fits || (!right.dimsKnown() && factor * right.getOutputMemEstimate() < OptimizerUtils.getRemoteMemBudgetMap( true))) // dims unknown but worst-case estimate fits { ret = true; } } return ret; }
/** * This will check if there is sufficient memory locally (twice the size of second matrix, for * original and sort data), and remotely (size of second matrix (sorted data)). * * @return true if sufficient memory */ private boolean isUnaryAggregateOuterSPRewriteApplicable() { boolean ret = false; Hop input = getInput().get(0); if (input instanceof BinaryOp && ((BinaryOp) input).isOuterVectorOperator()) { // note: both cases (partitioned matrix, and sorted double array), require to // fit the broadcast twice into the local memory budget. Also, the memory // constraint only needs to take the rhs into account because the output is // guaranteed to be an aggregate of <=16KB Hop right = input.getInput().get(1); double size = right.dimsKnown() ? OptimizerUtils.estimateSize(right.getDim1(), right.getDim2()) : // dims known and estimate fits right.getOutputMemEstimate(); // dims unknown but worst-case estimate fits if (_op == AggOp.MAXINDEX || _op == AggOp.MININDEX) { double memBudgetExec = SparkExecutionContext.getBroadcastMemoryBudget(); double memBudgetLocal = OptimizerUtils.getLocalMemBudget(); // basic requirement: the broadcast needs to to fit twice in the remote broadcast memory // and local memory budget because we have to create a partitioned broadcast // memory and hand it over to the spark context as in-memory object ret = (2 * size < memBudgetExec && 2 * size < memBudgetLocal); } else { if (OptimizerUtils.checkSparkBroadcastMemoryBudget(size)) { ret = true; } } } return ret; }
/** * @return * @throws HopsException * @throws LopsException */ private Lop constructLopsSparkCumulativeUnary() throws HopsException, LopsException { Hop input = getInput().get(0); long rlen = input.getDim1(); long clen = input.getDim2(); long brlen = input.getRowsInBlock(); long bclen = input.getColsInBlock(); boolean force = !dimsKnown() || _etypeForced == ExecType.SPARK; OperationTypes aggtype = getCumulativeAggType(); Lop X = input.constructLops(); Lop TEMP = X; ArrayList<Lop> DATA = new ArrayList<Lop>(); int level = 0; // recursive preaggregation until aggregates fit into CP memory budget while (((2 * OptimizerUtils.estimateSize(TEMP.getOutputParameters().getNumRows(), clen) + OptimizerUtils.estimateSize(1, clen)) > OptimizerUtils.getLocalMemBudget() && TEMP.getOutputParameters().getNumRows() > 1) || force) { DATA.add(TEMP); // preaggregation per block (for spark, the CumulativePartialAggregate subsumes both // the preaggregation and subsequent block aggregation) long rlenAgg = (long) Math.ceil((double) TEMP.getOutputParameters().getNumRows() / brlen); Lop preagg = new CumulativePartialAggregate( TEMP, DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.SPARK); preagg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1); setLineNumbers(preagg); TEMP = preagg; level++; force = false; // in case of unknowns, generate one level } // in-memory cum sum (of partial aggregates) if (TEMP.getOutputParameters().getNumRows() != 1) { int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads); Unary unary1 = new Unary( TEMP, HopsOpOp1LopsU.get(_op), DataType.MATRIX, ValueType.DOUBLE, ExecType.CP, k); unary1 .getOutputParameters() .setDimensions(TEMP.getOutputParameters().getNumRows(), clen, brlen, bclen, -1); setLineNumbers(unary1); TEMP = unary1; } // split, group and mr cumsum while (level-- > 0) { // (for spark, the CumulativeOffsetBinary subsumes both the split aggregate and // the subsequent offset binary apply of split aggregates against the original data) double initValue = getCumulativeInitValue(); CumulativeOffsetBinary binary = new CumulativeOffsetBinary( DATA.get(level), TEMP, DataType.MATRIX, ValueType.DOUBLE, initValue, aggtype, ExecType.SPARK); binary.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1); setLineNumbers(binary); TEMP = binary; } return TEMP; }
/** * MR Cumsum is currently based on a multipass algorithm of (1) preaggregation and (2) subsequent * offsetting. Note that we currently support one robust physical operator but many alternative * realizations are possible for specific scenarios (e.g., when the preaggregated intermediate fit * into the map task memory budget) or by creating custom job types. * * @return * @throws HopsException * @throws LopsException */ private Lop constructLopsMRCumulativeUnary() throws HopsException, LopsException { Hop input = getInput().get(0); long rlen = input.getDim1(); long clen = input.getDim2(); long brlen = input.getRowsInBlock(); long bclen = input.getColsInBlock(); boolean force = !dimsKnown() || _etypeForced == ExecType.MR; OperationTypes aggtype = getCumulativeAggType(); Lop X = input.constructLops(); Lop TEMP = X; ArrayList<Lop> DATA = new ArrayList<Lop>(); int level = 0; // recursive preaggregation until aggregates fit into CP memory budget while (((2 * OptimizerUtils.estimateSize(TEMP.getOutputParameters().getNumRows(), clen) + OptimizerUtils.estimateSize(1, clen)) > OptimizerUtils.getLocalMemBudget() && TEMP.getOutputParameters().getNumRows() > 1) || force) { DATA.add(TEMP); // preaggregation per block long rlenAgg = (long) Math.ceil((double) TEMP.getOutputParameters().getNumRows() / brlen); Lop preagg = new CumulativePartialAggregate( TEMP, DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.MR); preagg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1); setLineNumbers(preagg); Group group = new Group(preagg, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE); group.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1); setLineNumbers(group); Aggregate agg = new Aggregate( group, HopsAgg2Lops.get(AggOp.SUM), getDataType(), getValueType(), ExecType.MR); agg.getOutputParameters().setDimensions(rlenAgg, clen, brlen, bclen, -1); agg.setupCorrectionLocation( CorrectionLocationType .NONE); // aggregation uses kahanSum but the inputs do not have correction values setLineNumbers(agg); TEMP = agg; level++; force = false; // in case of unknowns, generate one level } // in-memory cum sum (of partial aggregates) if (TEMP.getOutputParameters().getNumRows() != 1) { int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads); Unary unary1 = new Unary( TEMP, HopsOpOp1LopsU.get(_op), DataType.MATRIX, ValueType.DOUBLE, ExecType.CP, k); unary1 .getOutputParameters() .setDimensions(TEMP.getOutputParameters().getNumRows(), clen, brlen, bclen, -1); setLineNumbers(unary1); TEMP = unary1; } // split, group and mr cumsum while (level-- > 0) { double init = getCumulativeInitValue(); CumulativeSplitAggregate split = new CumulativeSplitAggregate(TEMP, DataType.MATRIX, ValueType.DOUBLE, init); split.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1); setLineNumbers(split); Group group1 = new Group(DATA.get(level), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE); group1.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1); setLineNumbers(group1); Group group2 = new Group(split, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE); group2.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1); setLineNumbers(group2); CumulativeOffsetBinary binary = new CumulativeOffsetBinary( group1, group2, DataType.MATRIX, ValueType.DOUBLE, aggtype, ExecType.MR); binary.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, -1); setLineNumbers(binary); TEMP = binary; } return TEMP; }