/**
* This function will get slice of the input frame block overlapping in overall slice(Range),
* slice has requested for.
*
* @param val
* @param range
* @param brlen
* @param bclen
* @param outlist
* @throws DMLRuntimeException
*/
public static void performSlice(
Pair<Long, FrameBlock> in,
IndexRange ixrange,
int brlen,
int bclen,
ArrayList<Pair<Long, FrameBlock>> outlist)
throws DMLRuntimeException {
long index = in.getKey();
FrameBlock block = in.getValue();
// Get Block indexes (rows and columns boundaries)
long cellIndexTopRow = index;
long cellIndexBottomRow = index + block.getNumRows() - 1;
long cellIndexLeftCol = 1;
long cellIndexRightCol = block.getNumColumns();
// Calculate block boundaries with range of slice to be performed (Global index)
long cellIndexOverlapTop = Math.max(cellIndexTopRow, ixrange.rowStart);
long cellIndexOverlapBottom = Math.min(cellIndexBottomRow, ixrange.rowEnd);
long cellIndexOverlapLeft = Math.max(cellIndexLeftCol, ixrange.colStart);
long cellIndexOverlapRight = Math.min(cellIndexRightCol, ixrange.colEnd);
// check if block is outside the indexing range
if (cellIndexOverlapTop > cellIndexOverlapBottom
|| cellIndexOverlapLeft > cellIndexOverlapRight) {
return;
}
// Create IndexRange for the slice to be performed on this block.
IndexRange tmpRange =
new IndexRange(
cellIndexOverlapTop - index,
cellIndexOverlapBottom - index,
cellIndexOverlapLeft - 1,
cellIndexOverlapRight - 1);
// Get Top Row and Left column cutting point.
int rowCut = (int) (ixrange.rowStart - index);
// Get indices for result block
long resultBlockIndexTop = UtilFunctions.computeBlockIndex(cellIndexOverlapTop, brlen);
long resultBlockIndexBottom = UtilFunctions.computeBlockIndex(cellIndexOverlapBottom, brlen);
// allocate space for the output value
for (long r = resultBlockIndexTop; r <= resultBlockIndexBottom; r++) {
List<ValueType> schema =
UtilFunctions.getSubSchema(block.getSchema(), tmpRange.colStart, tmpRange.colEnd);
long iResultIndex = (r - 1) * brlen + tmpRange.rowStart;
Pair<Long, FrameBlock> out =
new Pair<Long, FrameBlock>(new Long(iResultIndex + 1), new FrameBlock(schema));
outlist.add(out);
}
// execute actual slice operation
block.sliceOperations(outlist, tmpRange, rowCut);
}
@Override
public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(
Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception {
ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret =
new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>();
MatrixIndexes ixIn = arg0._1();
MatrixBlock mb2 = arg0._2();
// get the right hand side matrix
MatrixBlock mb1 = _pmV.getMatrixBlock((int) ixIn.getRowIndex(), 1);
// compute target block indexes
long minPos = UtilFunctions.toLong(mb1.minNonZero());
long maxPos = UtilFunctions.toLong(mb1.max());
long rowIX1 = (minPos - 1) / _brlen + 1;
long rowIX2 = (maxPos - 1) / _brlen + 1;
boolean multipleOuts = (rowIX1 != rowIX2);
if (minPos >= 1) // at least one row selected
{
// output sparsity estimate
double spmb1 = OptimizerUtils.getSparsity(mb1.getNumRows(), 1, mb1.getNonZeros());
long estnnz = (long) (spmb1 * mb2.getNonZeros());
boolean sparse = MatrixBlock.evalSparseFormatInMemory(_brlen, mb2.getNumColumns(), estnnz);
// compute and allocate output blocks
MatrixBlock out1 = new MatrixBlock();
MatrixBlock out2 = multipleOuts ? new MatrixBlock() : null;
out1.reset(_brlen, mb2.getNumColumns(), sparse);
if (out2 != null)
out2.reset(
UtilFunctions.computeBlockSize(_rlen, rowIX2, _brlen), mb2.getNumColumns(), sparse);
// compute core matrix permutation (assumes that out1 has default blocksize,
// hence we do a meta data correction afterwards)
mb1.permutationMatrixMultOperations(mb2, out1, out2);
out1.setNumRows(UtilFunctions.computeBlockSize(_rlen, rowIX1, _brlen));
ret.add(
new Tuple2<MatrixIndexes, MatrixBlock>(
new MatrixIndexes(rowIX1, ixIn.getColumnIndex()), out1));
if (out2 != null)
ret.add(
new Tuple2<MatrixIndexes, MatrixBlock>(
new MatrixIndexes(rowIX2, ixIn.getColumnIndex()), out2));
}
return ret;
}
public ScalarObject getScalarInput(String name, ValueType vt, boolean isLiteral)
throws DMLRuntimeException {
if (isLiteral) {
switch (vt) {
case INT:
long intVal = UtilFunctions.parseToLong(name);
IntObject intObj = new IntObject(intVal);
return intObj;
case DOUBLE:
double doubleVal = Double.parseDouble(name);
DoubleObject doubleObj = new DoubleObject(doubleVal);
return doubleObj;
case BOOLEAN:
Boolean boolVal = Boolean.parseBoolean(name);
BooleanObject boolObj = new BooleanObject(boolVal);
return boolObj;
case STRING:
StringObject stringObj = new StringObject(name);
return stringObj;
default:
throw new DMLRuntimeException("Unknown value type: " + vt + " for variable: " + name);
}
} else {
Data obj = getVariable(name);
if (obj == null) {
throw new DMLRuntimeException("Unknown variable: " + name);
}
return (ScalarObject) obj;
}
}
/**
* @param sqlctx
* @param mb
* @param schema
* @return
* @throws DMLRuntimeException
*/
@SuppressWarnings("resource")
private DataFrame createDataFrame(
SQLContext sqlctx, MatrixBlock mb, boolean containsID, ValueType[] schema)
throws DMLRuntimeException {
// create in-memory list of rows
List<Row> list = new ArrayList<Row>();
int off = (containsID ? 1 : 0);
int clen = mb.getNumColumns() + off - colsVector + 1;
for (int i = 0; i < mb.getNumRows(); i++) {
Object[] row = new Object[clen];
if (containsID) row[0] = i + 1;
for (int j = 0, j2 = 0; j < mb.getNumColumns(); j++, j2++) {
if (schema[j2] != ValueType.OBJECT) {
row[j2 + off] = UtilFunctions.doubleToObject(schema[j2], mb.quickGetValue(i, j));
} else {
double[] tmp =
DataConverter.convertToDoubleVector(
mb.sliceOperations(i, i, j, j + colsVector - 1, new MatrixBlock()));
row[j2 + off] = new DenseVector(tmp);
j += colsVector - 1;
}
}
list.add(RowFactory.create(row));
}
// create data frame schema
List<StructField> fields = new ArrayList<StructField>();
if (containsID)
fields.add(
DataTypes.createStructField(RDDConverterUtils.DF_ID_COLUMN, DataTypes.DoubleType, true));
for (int j = 0; j < schema.length; j++) {
DataType dt = null;
switch (schema[j]) {
case STRING:
dt = DataTypes.StringType;
break;
case DOUBLE:
dt = DataTypes.DoubleType;
break;
case INT:
dt = DataTypes.LongType;
break;
case OBJECT:
dt = new VectorUDT();
break;
default:
throw new RuntimeException("Unsupported value type.");
}
fields.add(DataTypes.createStructField("C" + (j + 1), dt, true));
}
StructType dfSchema = DataTypes.createStructType(fields);
// create rdd and data frame
JavaSparkContext sc = new JavaSparkContext(sqlctx.sparkContext());
JavaRDD<Row> rowRDD = sc.parallelize(list);
return sqlctx.createDataFrame(rowRDD, dfSchema);
}
// Reused by both MR and Spark for performing zero out
public static IndexRange getSelectedRangeForZeroOut(
Pair<Long, FrameBlock> in,
int blockRowFactor,
int blockColFactor,
IndexRange indexRange,
long lSrcRowIndex,
long lDestRowIndex) {
int iRowStart, iRowEnd, iColStart, iColEnd;
if (indexRange.rowStart <= lDestRowIndex) iRowStart = 0;
else iRowStart = (int) (indexRange.rowStart - in.getKey());
iRowEnd = (int) Math.min(indexRange.rowEnd - lSrcRowIndex, blockRowFactor) - 1;
iColStart = UtilFunctions.computeCellInBlock(indexRange.colStart, blockColFactor);
iColEnd = UtilFunctions.computeCellInBlock(indexRange.colEnd, blockColFactor);
return new IndexRange(iRowStart, iRowEnd, iColStart, iColEnd);
}
/**
* @param vector
* @param singleColBlock
* @param dense
* @param unknownDims
*/
private void testDataFrameConversion(
ValueType[] schema, boolean containsID, boolean dense, boolean unknownDims) {
boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform;
SparkExecutionContext sec = null;
try {
DMLScript.USE_LOCAL_SPARK_CONFIG = true;
DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
// generate input data and setup metadata
int cols = schema.length + colsVector - 1;
double sparsity = dense ? sparsity1 : sparsity2;
double[][] A = TestUtils.round(getRandomMatrix(rows1, cols, -10, 1000, sparsity, 2373));
MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc1 =
new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros());
MatrixCharacteristics mc2 =
unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
// setup spark context
sec = (SparkExecutionContext) ExecutionContextFactory.createContext();
JavaSparkContext sc = sec.getSparkContext();
SQLContext sqlctx = new SQLContext(sc);
// create input data frame
DataFrame df = createDataFrame(sqlctx, mbA, containsID, schema);
// dataframe - frame conversion
JavaPairRDD<Long, FrameBlock> out =
FrameRDDConverterUtils.dataFrameToBinaryBlock(sc, df, mc2, containsID);
// get output frame block
FrameBlock fbB =
SparkExecutionContext.toFrameBlock(
out, UtilFunctions.nCopies(cols, ValueType.DOUBLE), rows1, cols);
// compare frame blocks
MatrixBlock mbB = DataConverter.convertToMatrixBlock(fbB);
double[][] B = DataConverter.convertToDoubleMatrix(mbB);
TestUtils.compareMatrices(A, B, rows1, cols, eps);
} catch (Exception ex) {
throw new RuntimeException(ex);
} finally {
sec.close();
DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig;
DMLScript.rtplatform = oldPlatform;
}
}
/**
* @param vt
* @param in
* @param sparse
* @return
*/
public static Object doubleToObject(ValueType vt, double in, boolean sparse) {
if (in == 0 && sparse) return null;
switch (vt) {
case STRING:
return String.valueOf(in);
case BOOLEAN:
return (in != 0);
case INT:
return UtilFunctions.toLong(in);
case DOUBLE:
return in;
default:
throw new RuntimeException("Unsupported value type: " + vt);
}
}
@Override
public MatrixBlock apply(FrameBlock in, MatrixBlock out) {
for (int j = 0; j < _colList.length; j++) {
int col = _colList[j] - 1;
ValueType vt = in.getSchema()[col];
for (int i = 0; i < in.getNumRows(); i++) {
Object val = in.get(i, col);
out.quickSetValue(
i,
col,
(val == null || (vt == ValueType.STRING && val.toString().isEmpty()))
? Double.NaN
: UtilFunctions.objectToDouble(vt, val));
}
}
return out;
}
// Reused by both MR and Spark for performing zero out
public static IndexRange getSelectedRangeForZeroOut(
IndexedMatrixValue in, int blockRowFactor, int blockColFactor, IndexRange indexRange) {
IndexRange tempRange = new IndexRange(-1, -1, -1, -1);
long topBlockRowIndex = UtilFunctions.computeBlockIndex(indexRange.rowStart, blockRowFactor);
int topRowInTopBlock = UtilFunctions.computeCellInBlock(indexRange.rowStart, blockRowFactor);
long bottomBlockRowIndex = UtilFunctions.computeBlockIndex(indexRange.rowEnd, blockRowFactor);
int bottomRowInBottomBlock =
UtilFunctions.computeCellInBlock(indexRange.rowEnd, blockRowFactor);
long leftBlockColIndex = UtilFunctions.computeBlockIndex(indexRange.colStart, blockColFactor);
int leftColInLeftBlock = UtilFunctions.computeCellInBlock(indexRange.colStart, blockColFactor);
long rightBlockColIndex = UtilFunctions.computeBlockIndex(indexRange.colEnd, blockColFactor);
int rightColInRightBlock = UtilFunctions.computeCellInBlock(indexRange.colEnd, blockColFactor);
// no overlap
if (in.getIndexes().getRowIndex() < topBlockRowIndex
|| in.getIndexes().getRowIndex() > bottomBlockRowIndex
|| in.getIndexes().getColumnIndex() < leftBlockColIndex
|| in.getIndexes().getColumnIndex() > rightBlockColIndex) {
tempRange.set(-1, -1, -1, -1);
return tempRange;
}
// get the index range inside the block
tempRange.set(0, in.getValue().getNumRows() - 1, 0, in.getValue().getNumColumns() - 1);
if (topBlockRowIndex == in.getIndexes().getRowIndex()) tempRange.rowStart = topRowInTopBlock;
if (bottomBlockRowIndex == in.getIndexes().getRowIndex())
tempRange.rowEnd = bottomRowInBottomBlock;
if (leftBlockColIndex == in.getIndexes().getColumnIndex())
tempRange.colStart = leftColInLeftBlock;
if (rightBlockColIndex == in.getIndexes().getColumnIndex())
tempRange.colEnd = rightColInRightBlock;
return tempRange;
}
@Override
public int hashCode() {
return UtilFunctions.longlongHashCode(_row, _col);
}
/**
* @param in
* @param ixrange
* @param brlen
* @param bclen
* @param rlen
* @param clen
* @param outlist
* @throws DMLRuntimeException
*/
public static void performShift(
Pair<Long, FrameBlock> in,
IndexRange ixrange,
int brlenLeft,
int clenLeft /*, int bclen*/,
long rlen,
long clen,
ArrayList<Pair<Long, FrameBlock>> outlist)
throws DMLRuntimeException {
Long ix = in.getKey();
FrameBlock fb = in.getValue();
long start_lhs_globalRowIndex = ixrange.rowStart + (ix - 1);
long start_lhs_globalColIndex = ixrange.colStart;
long end_lhs_globalRowIndex = start_lhs_globalRowIndex + fb.getNumRows() - 1;
long end_lhs_globalColIndex = ixrange.colEnd;
long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlenLeft);
long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlenLeft);
for (long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
// Calculate global index of right hand side block
long lhs_rl = Math.max((leftRowIndex - 1) * brlenLeft + 1, start_lhs_globalRowIndex);
long lhs_ru = Math.min(leftRowIndex * brlenLeft, end_lhs_globalRowIndex);
long lhs_cl = start_lhs_globalColIndex;
long lhs_cu = end_lhs_globalColIndex;
int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlenLeft);
int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlenLeft);
int lhs_lcl = (int) lhs_cl - 1;
int lhs_lcu = (int) lhs_cu - 1;
long rhs_rl = lhs_rl - (ixrange.rowStart - 1) - (ix - 1);
long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
long rhs_cl = lhs_cl - ixrange.colStart + 1;
long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
// local indices are 0 (zero) based.
int rhs_lrl = (int) (UtilFunctions.computeCellInBlock(rhs_rl, fb.getNumRows()));
int rhs_lru = (int) (UtilFunctions.computeCellInBlock(rhs_ru, fb.getNumRows()));
int rhs_lcl = (int) rhs_cl - 1;
int rhs_lcu = (int) rhs_cu - 1;
FrameBlock slicedRHSBlk =
fb.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new FrameBlock());
int lbclen = clenLeft;
List<ValueType> schemaPartialLeft = Collections.nCopies(lhs_lcl, ValueType.STRING);
List<ValueType> schemaRHS =
UtilFunctions.getSubSchema(fb.getSchema(), rhs_lcl, rhs_lcl - lhs_lcl + lhs_lcu);
List<ValueType> schema = new ArrayList<ValueType>(schemaPartialLeft);
schema.addAll(schemaRHS);
List<ValueType> schemaPartialRight =
Collections.nCopies(lbclen - schema.size(), ValueType.STRING);
schema.addAll(schemaPartialRight);
FrameBlock resultBlock = new FrameBlock(schema);
int iRHSRows =
(int)
(leftRowIndex <= rlen / brlenLeft
? brlenLeft
: rlen - (rlen / brlenLeft) * brlenLeft);
resultBlock.ensureAllocatedColumns(iRHSRows);
resultBlock =
resultBlock.leftIndexingOperations(
slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, new FrameBlock());
outlist.add(new Pair<Long, FrameBlock>((leftRowIndex - 1) * brlenLeft + 1, resultBlock));
}
}
/**
* @param in
* @param ixrange
* @param brlen
* @param bclen
* @param rlen
* @param clen
* @param outlist
* @throws DMLRuntimeException
*/
public static void performShift(
IndexedMatrixValue in,
IndexRange ixrange,
int brlen,
int bclen,
long rlen,
long clen,
ArrayList<IndexedMatrixValue> outlist)
throws DMLRuntimeException {
MatrixIndexes ix = in.getIndexes();
MatrixBlock mb = (MatrixBlock) in.getValue();
long start_lhs_globalRowIndex = ixrange.rowStart + (ix.getRowIndex() - 1) * brlen;
long start_lhs_globalColIndex = ixrange.colStart + (ix.getColumnIndex() - 1) * bclen;
long end_lhs_globalRowIndex = start_lhs_globalRowIndex + mb.getNumRows() - 1;
long end_lhs_globalColIndex = start_lhs_globalColIndex + mb.getNumColumns() - 1;
long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen);
long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen);
long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen);
long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen);
for (long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
for (long leftColIndex = start_lhs_colIndex;
leftColIndex <= end_lhs_colIndex;
leftColIndex++) {
// Calculate global index of right hand side block
long lhs_rl = Math.max((leftRowIndex - 1) * brlen + 1, start_lhs_globalRowIndex);
long lhs_ru = Math.min(leftRowIndex * brlen, end_lhs_globalRowIndex);
long lhs_cl = Math.max((leftColIndex - 1) * bclen + 1, start_lhs_globalColIndex);
long lhs_cu = Math.min(leftColIndex * bclen, end_lhs_globalColIndex);
int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen);
int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen);
int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen);
int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen);
long rhs_rl = lhs_rl - ixrange.rowStart + 1;
long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
long rhs_cl = lhs_cl - ixrange.colStart + 1;
long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen);
int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen);
int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen);
int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen);
MatrixBlock slicedRHSBlk =
mb.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock());
int lbrlen = UtilFunctions.computeBlockSize(rlen, leftRowIndex, brlen);
int lbclen = UtilFunctions.computeBlockSize(clen, leftColIndex, bclen);
MatrixBlock resultBlock = new MatrixBlock(lbrlen, lbclen, false);
resultBlock =
resultBlock.leftIndexingOperations(
slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, UpdateType.COPY);
outlist.add(
new IndexedMatrixValue(new MatrixIndexes(leftRowIndex, leftColIndex), resultBlock));
}
}
}
/**
* @param val
* @param range
* @param brlen
* @param bclen
* @param outlist
* @throws DMLRuntimeException
*/
public static void performSlice(
IndexedMatrixValue in,
IndexRange ixrange,
int brlen,
int bclen,
ArrayList<IndexedMatrixValue> outlist)
throws DMLRuntimeException {
long cellIndexTopRow = UtilFunctions.computeCellIndex(in.getIndexes().getRowIndex(), brlen, 0);
long cellIndexBottomRow =
UtilFunctions.computeCellIndex(
in.getIndexes().getRowIndex(), brlen, in.getValue().getNumRows() - 1);
long cellIndexLeftCol =
UtilFunctions.computeCellIndex(in.getIndexes().getColumnIndex(), bclen, 0);
long cellIndexRightCol =
UtilFunctions.computeCellIndex(
in.getIndexes().getColumnIndex(), bclen, in.getValue().getNumColumns() - 1);
long cellIndexOverlapTop = Math.max(cellIndexTopRow, ixrange.rowStart);
long cellIndexOverlapBottom = Math.min(cellIndexBottomRow, ixrange.rowEnd);
long cellIndexOverlapLeft = Math.max(cellIndexLeftCol, ixrange.colStart);
long cellIndexOverlapRight = Math.min(cellIndexRightCol, ixrange.colEnd);
// check if block is outside the indexing range
if (cellIndexOverlapTop > cellIndexOverlapBottom
|| cellIndexOverlapLeft > cellIndexOverlapRight) {
return;
}
IndexRange tmpRange =
new IndexRange(
UtilFunctions.computeCellInBlock(cellIndexOverlapTop, brlen),
UtilFunctions.computeCellInBlock(cellIndexOverlapBottom, brlen),
UtilFunctions.computeCellInBlock(cellIndexOverlapLeft, bclen),
UtilFunctions.computeCellInBlock(cellIndexOverlapRight, bclen));
int rowCut = UtilFunctions.computeCellInBlock(ixrange.rowStart, brlen);
int colCut = UtilFunctions.computeCellInBlock(ixrange.colStart, bclen);
int rowsInLastBlock = (int) ((ixrange.rowEnd - ixrange.rowStart + 1) % brlen);
if (rowsInLastBlock == 0) rowsInLastBlock = brlen;
int colsInLastBlock = (int) ((ixrange.colEnd - ixrange.colStart + 1) % bclen);
if (colsInLastBlock == 0) colsInLastBlock = bclen;
long resultBlockIndexTop =
UtilFunctions.computeBlockIndex(cellIndexOverlapTop - ixrange.rowStart + 1, brlen);
long resultBlockIndexBottom =
UtilFunctions.computeBlockIndex(cellIndexOverlapBottom - ixrange.rowStart + 1, brlen);
long resultBlockIndexLeft =
UtilFunctions.computeBlockIndex(cellIndexOverlapLeft - ixrange.colStart + 1, bclen);
long resultBlockIndexRight =
UtilFunctions.computeBlockIndex(cellIndexOverlapRight - ixrange.colStart + 1, bclen);
int boundaryRlen = brlen;
int boundaryClen = bclen;
long finalBlockIndexBottom =
UtilFunctions.computeBlockIndex(ixrange.rowEnd - ixrange.rowStart + 1, brlen);
long finalBlockIndexRight =
UtilFunctions.computeBlockIndex(ixrange.colEnd - ixrange.colStart + 1, bclen);
if (resultBlockIndexBottom == finalBlockIndexBottom) boundaryRlen = rowsInLastBlock;
if (resultBlockIndexRight == finalBlockIndexRight) boundaryClen = colsInLastBlock;
// allocate space for the output value
for (long r = resultBlockIndexTop; r <= resultBlockIndexBottom; r++)
for (long c = resultBlockIndexLeft; c <= resultBlockIndexRight; c++) {
IndexedMatrixValue out = new IndexedMatrixValue(new MatrixIndexes(), new MatrixBlock());
out.getIndexes().setIndexes(r, c);
outlist.add(out);
}
// execute actual slice operation
in.getValue()
.sliceOperations(
outlist, tmpRange, rowCut, colCut, brlen, bclen, boundaryRlen, boundaryClen);
}
public static long getLengthForInterQuantile(NumItemsByEachReducerMetaData metadata, double p) {
long total = UtilFunctions.getTotalLength(metadata);
long lpos = (long) Math.ceil(total * p); // lower bound is inclusive
long upos = (long) Math.ceil(total * (1 - p)); // upper bound is inclusive
return upos - lpos + 1;
}
