@Override public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception { MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); MatrixIndexes ixOut = new MatrixIndexes(); MatrixBlock blkOut = new MatrixBlock(); if (_type == CacheType.LEFT) { // get the right hand side matrix MatrixBlock left = _pbc.getMatrixBlock(1, (int) ixIn.getRowIndex()); // execute matrix-vector mult OperationsOnMatrixValues.performAggregateBinary( new MatrixIndexes(1, ixIn.getRowIndex()), left, ixIn, blkIn, ixOut, blkOut, _op); } else // if( _type == CacheType.RIGHT ) { // get the right hand side matrix MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), 1); // execute matrix-vector mult OperationsOnMatrixValues.performAggregateBinary( ixIn, blkIn, new MatrixIndexes(ixIn.getColumnIndex(), 1), right, ixOut, blkOut, _op); } // output new tuple return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut); }
public static void performAggregateBinary( MatrixIndexes indexes1, MatrixValue value1, MatrixIndexes indexes2, MatrixValue value2, MatrixIndexes indexesOut, MatrixValue valueOut, AggregateBinaryOperator op) throws DMLRuntimeException { // compute output index indexesOut.setIndexes(indexes1.getRowIndex(), indexes2.getColumnIndex()); // perform on the value value1.aggregateBinaryOperations(indexes1, value1, indexes2, value2, valueOut, op); }
/** * @param ix * @param brlen * @param bclen * @param rl * @param ru * @param cl * @param cu * @return */ public static boolean isInBlockRange( MatrixIndexes ix, int brlen, int bclen, long rl, long ru, long cl, long cu) { long bRLowerIndex = (ix.getRowIndex() - 1) * brlen + 1; long bRUpperIndex = ix.getRowIndex() * brlen; long bCLowerIndex = (ix.getColumnIndex() - 1) * bclen + 1; long bCUpperIndex = ix.getColumnIndex() * bclen; if (rl > bRUpperIndex || ru < bRLowerIndex) { return false; } else if (cl > bCUpperIndex || cu < bCLowerIndex) { return false; } else { return true; } }
@Override public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call( Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception { ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>(); MatrixIndexes ixIn = arg0._1(); MatrixBlock mb2 = arg0._2(); // get the right hand side matrix MatrixBlock mb1 = _pmV.getMatrixBlock((int) ixIn.getRowIndex(), 1); // compute target block indexes long minPos = UtilFunctions.toLong(mb1.minNonZero()); long maxPos = UtilFunctions.toLong(mb1.max()); long rowIX1 = (minPos - 1) / _brlen + 1; long rowIX2 = (maxPos - 1) / _brlen + 1; boolean multipleOuts = (rowIX1 != rowIX2); if (minPos >= 1) // at least one row selected { // output sparsity estimate double spmb1 = OptimizerUtils.getSparsity(mb1.getNumRows(), 1, mb1.getNonZeros()); long estnnz = (long) (spmb1 * mb2.getNonZeros()); boolean sparse = MatrixBlock.evalSparseFormatInMemory(_brlen, mb2.getNumColumns(), estnnz); // compute and allocate output blocks MatrixBlock out1 = new MatrixBlock(); MatrixBlock out2 = multipleOuts ? new MatrixBlock() : null; out1.reset(_brlen, mb2.getNumColumns(), sparse); if (out2 != null) out2.reset( UtilFunctions.computeBlockSize(_rlen, rowIX2, _brlen), mb2.getNumColumns(), sparse); // compute core matrix permutation (assumes that out1 has default blocksize, // hence we do a meta data correction afterwards) mb1.permutationMatrixMultOperations(mb2, out1, out2); out1.setNumRows(UtilFunctions.computeBlockSize(_rlen, rowIX1, _brlen)); ret.add( new Tuple2<MatrixIndexes, MatrixBlock>( new MatrixIndexes(rowIX1, ixIn.getColumnIndex()), out1)); if (out2 != null) ret.add( new Tuple2<MatrixIndexes, MatrixBlock>( new MatrixIndexes(rowIX2, ixIn.getColumnIndex()), out2)); } return ret; }
@Override public void processInstruction( Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) throws DMLUnsupportedOperationException, DMLRuntimeException { ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input); if (blkList != null) for (IndexedMatrixValue in : blkList) { if (in == null) continue; // allocate space for the output value IndexedMatrixValue out; if (input == output) out = tempValue; else out = cachedValues.holdPlace(output, valueClass); MatrixIndexes inix = in.getIndexes(); // prune unnecessary blocks for trace if ((((AggregateUnaryOperator) optr).indexFn instanceof ReduceDiag && inix.getColumnIndex() != inix.getRowIndex())) { // do nothing (block not on diagonal); but reset out.getValue().reset(); } else // general case { // process instruction AggregateUnaryOperator auop = (AggregateUnaryOperator) optr; OperationsOnMatrixValues.performAggregateUnary( inix, in.getValue(), out.getIndexes(), out.getValue(), auop, blockRowFactor, blockColFactor); if (_dropCorr) ((MatrixBlock) out.getValue()).dropLastRowsOrColums(auop.aggOp.correctionLocation); } // put the output value in the cache if (out == tempValue) cachedValues.add(output, out); } }
@Override public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception { MatrixBlock pmV = _pmV.getMatrixBlock(1, 1); MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); int rowIx = (int) ixIn.getRowIndex(); MatrixIndexes ixOut = new MatrixIndexes(1, 1); MatrixBlock blkOut = new MatrixBlock(); // execute mapmmchain operation blkIn.chainMatrixMultOperations(pmV, _pmW.getMatrixBlock(rowIx, 1), blkOut, ChainType.XtwXv); // output new tuple return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut); }
@Override public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call( Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception { ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>(); MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); if (_type == CacheType.LEFT) { // for all matching left-hand-side blocks int len = _pbc.getNumRowBlocks(); for (int i = 1; i <= len; i++) { MatrixBlock left = _pbc.getMatrixBlock(i, (int) ixIn.getRowIndex()); MatrixIndexes ixOut = new MatrixIndexes(); MatrixBlock blkOut = new MatrixBlock(); // execute matrix-vector mult OperationsOnMatrixValues.performAggregateBinary( new MatrixIndexes(i, ixIn.getRowIndex()), left, ixIn, blkIn, ixOut, blkOut, _op); ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut)); } } else // if( _type == CacheType.RIGHT ) { // for all matching right-hand-side blocks int len = _pbc.getNumColumnBlocks(); for (int j = 1; j <= len; j++) { // get the right hand side matrix MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), j); MatrixIndexes ixOut = new MatrixIndexes(); MatrixBlock blkOut = new MatrixBlock(); // execute matrix-vector mult OperationsOnMatrixValues.performAggregateBinary( ixIn, blkIn, new MatrixIndexes(ixIn.getColumnIndex(), j), right, ixOut, blkOut, _op); ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut)); } } return ret; }
public static void performZeroOut( MatrixIndexes indexesIn, MatrixValue valueIn, MatrixIndexes indexesOut, MatrixValue valueOut, IndexRange range, boolean complementary) throws DMLRuntimeException { valueIn.zeroOutOperations(valueOut, range, complementary); indexesOut.setIndexes(indexesIn); }
/** * @param target * @param groups * @param brlen * @param bclen * @param outlist * @throws DMLRuntimeException */ public static void performMapGroupedAggregate( Operator op, IndexedMatrixValue inTarget, MatrixBlock groups, int ngroups, int brlen, int bclen, ArrayList<IndexedMatrixValue> outlist) throws DMLRuntimeException { MatrixIndexes ix = inTarget.getIndexes(); MatrixBlock target = (MatrixBlock) inTarget.getValue(); // execute grouped aggregate operations MatrixBlock out = groups.groupedAggOperations(target, null, new MatrixBlock(), ngroups, op); if (out.getNumRows() <= brlen && out.getNumColumns() <= bclen) { // single output block outlist.add(new IndexedMatrixValue(new MatrixIndexes(1, ix.getColumnIndex()), out)); } else { // multiple output blocks (by op def, single column block ) for (int blockRow = 0; blockRow < (int) Math.ceil(out.getNumRows() / (double) brlen); blockRow++) { int maxRow = (blockRow * brlen + brlen < out.getNumRows()) ? brlen : out.getNumRows() - blockRow * brlen; int row_offset = blockRow * brlen; // copy submatrix to block MatrixBlock tmp = out.sliceOperations( row_offset, row_offset + maxRow - 1, 0, out.getNumColumns() - 1, new MatrixBlock()); // append block to result cache outlist.add( new IndexedMatrixValue(new MatrixIndexes(blockRow + 1, ix.getColumnIndex()), tmp)); } } }
@Override public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception { MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); MatrixIndexes ixOut = new MatrixIndexes(); MatrixBlock blkOut = new MatrixBlock(); // process instruction OperationsOnMatrixValues.performAggregateUnary( ixIn, blkIn, ixOut, blkOut, ((AggregateUnaryOperator) _op), _brlen, _bclen); if (((AggregateUnaryOperator) _op).aggOp.correctionExists) blkOut.dropLastRowsOrColums(((AggregateUnaryOperator) _op).aggOp.correctionLocation); // cumsum expand partial aggregates long rlenOut = (long) Math.ceil((double) _rlen / _brlen); long rixOut = (long) Math.ceil((double) ixIn.getRowIndex() / _brlen); int rlenBlk = (int) Math.min(rlenOut - (rixOut - 1) * _brlen, _brlen); int clenBlk = blkOut.getNumColumns(); int posBlk = (int) ((ixIn.getRowIndex() - 1) % _brlen); MatrixBlock blkOut2 = new MatrixBlock(rlenBlk, clenBlk, false); blkOut2.copy(posBlk, posBlk, 0, clenBlk - 1, blkOut, true); ixOut.setIndexes(rixOut, ixOut.getColumnIndex()); // output new tuple return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut2); }
@Override protected Tuple2<MatrixIndexes, MatrixBlock> computeNext( Tuple2<MatrixIndexes, MatrixBlock> arg) throws Exception { MatrixIndexes ixIn = arg._1(); MatrixBlock blkIn = arg._2(); MatrixBlock blkOut = new MatrixBlock(); if (_type == CacheType.LEFT) { // get the right hand side matrix MatrixBlock left = _pbc.getMatrixBlock(1, (int) ixIn.getRowIndex()); // execute index preserving matrix multiplication left.aggregateBinaryOperations(left, blkIn, blkOut, _op); } else // if( _type == CacheType.RIGHT ) { // get the right hand side matrix MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), 1); // execute index preserving matrix multiplication blkIn.aggregateBinaryOperations(blkIn, right, blkOut, _op); } return new Tuple2<MatrixIndexes, MatrixBlock>(ixIn, blkOut); }
@Override public void processInstruction( Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) throws DMLRuntimeException { ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input); if (blkList == null) return; for (IndexedMatrixValue in1 : blkList) { if (in1 == null) continue; MatrixIndexes inix = in1.getIndexes(); MatrixBlock blk = (MatrixBlock) in1.getValue(); long rixOffset = (inix.getRowIndex() - 1) * blockRowFactor; boolean firstBlk = (inix.getRowIndex() == 1); boolean lastBlk = (inix.getRowIndex() == _lastRowBlockIndex); // introduce offsets w/ init value for first row if (firstBlk) { IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass); ((MatrixBlock) out.getValue()).reset(1, blk.getNumColumns()); if (_initValue != 0) { for (int j = 0; j < blk.getNumColumns(); j++) ((MatrixBlock) out.getValue()).appendValue(0, j, _initValue); } out.getIndexes().setIndexes(1, inix.getColumnIndex()); } // output splitting (shift by one), preaggregated offset used by subsequent block for (int i = 0; i < blk.getNumRows(); i++) if (!(lastBlk && i == (blk.getNumRows() - 1))) // ignore last row { IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass); MatrixBlock tmpBlk = (MatrixBlock) out.getValue(); tmpBlk.reset(1, blk.getNumColumns()); blk.sliceOperations(i, i, 0, blk.getNumColumns() - 1, tmpBlk); out.getIndexes().setIndexes(rixOffset + i + 2, inix.getColumnIndex()); } } }
/** * @param path * @param job * @param fs * @param dest * @param rlen * @param clen * @param brlen * @param bclen * @throws IOException */ @SuppressWarnings("deprecation") private void readBinaryCellMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, long rlen, long clen, int brlen, int bclen) throws IOException { boolean sparse = dest.isInSparseFormat(); MatrixIndexes key = new MatrixIndexes(); MatrixCell value = new MatrixCell(); int row = -1; int col = -1; try { for (Path lpath : getSequenceFilePaths(fs, path)) // 1..N files { // directly read from sequence files (individual partfiles) SequenceFile.Reader reader = new SequenceFile.Reader(fs, lpath, job); try { if (sparse) { while (reader.next(key, value)) { row = (int) key.getRowIndex() - 1; col = (int) key.getColumnIndex() - 1; double lvalue = value.getValue(); dest.appendValue(row, col, lvalue); } } else { while (reader.next(key, value)) { row = (int) key.getRowIndex() - 1; col = (int) key.getColumnIndex() - 1; double lvalue = value.getValue(); dest.appendValue(row, col, lvalue); } } } finally { IOUtilFunctions.closeSilently(reader); } } if (sparse) dest.sortSparseRows(); } catch (Exception ex) { // post-mortem error handling and bounds checking if (row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen) { throw new IOException( "Matrix cell [" + (row + 1) + "," + (col + 1) + "] " + "out of overall matrix range [1:" + rlen + ",1:" + clen + "]."); } else { throw new IOException("Unable to read matrix in binary cell format.", ex); } } }
@Override public void processInstruction( Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) throws DMLRuntimeException { QuaternaryOperator qop = (QuaternaryOperator) optr; ArrayList<IndexedMatrixValue> blkList = cachedValues.get(_input1); if (blkList != null) for (IndexedMatrixValue imv : blkList) { // Step 1: prepare inputs and output if (imv == null) continue; MatrixIndexes inIx = imv.getIndexes(); MatrixValue inVal = imv.getValue(); // allocate space for the output value IndexedMatrixValue iout = null; if (output == _input1) iout = tempValue; else iout = cachedValues.holdPlace(output, valueClass); MatrixIndexes outIx = iout.getIndexes(); MatrixValue outVal = iout.getValue(); // Step 2: get remaining inputs: Wij, Ui, Vj MatrixValue Xij = inVal; // get Wij if existing (null of WeightsType.NONE or WSigmoid any type) IndexedMatrixValue iWij = (_input4 != -1) ? cachedValues.getFirst(_input4) : null; MatrixValue Wij = (iWij != null) ? iWij.getValue() : null; if (null == Wij && qop.hasFourInputs()) { MatrixBlock mb = new MatrixBlock(1, 1, false); String[] parts = InstructionUtils.getInstructionParts(instString); mb.quickSetValue(0, 0, Double.valueOf(parts[4])); Wij = mb; } // get Ui and Vj, potentially through distributed cache MatrixValue Ui = (!_cacheU) ? cachedValues.getFirst(_input2).getValue() // U : MRBaseForCommonInstructions.dcValues .get(_input2) .getDataBlock((int) inIx.getRowIndex(), 1) .getValue(); MatrixValue Vj = (!_cacheV) ? cachedValues.getFirst(_input3).getValue() // t(V) : MRBaseForCommonInstructions.dcValues .get(_input3) .getDataBlock((int) inIx.getColumnIndex(), 1) .getValue(); // handle special input case: //V through shuffle -> t(V) if (Ui.getNumColumns() != Vj.getNumColumns()) { Vj = LibMatrixReorg.reorg( (MatrixBlock) Vj, new MatrixBlock(Vj.getNumColumns(), Vj.getNumRows(), Vj.isInSparseFormat()), new ReorgOperator(SwapIndex.getSwapIndexFnObject())); } // Step 3: process instruction Xij.quaternaryOperations(qop, Ui, Vj, Wij, outVal); // set output indexes if (qop.wtype1 != null || qop.wtype4 != null) outIx.setIndexes(1, 1); // wsloss else if (qop.wtype2 != null || qop.wtype5 != null || qop.wtype3 != null && qop.wtype3.isBasic()) outIx.setIndexes(inIx); // wsigmoid/wdivmm-basic else { // wdivmm boolean left = qop.wtype3.isLeft(); outIx.setIndexes(left ? inIx.getColumnIndex() : inIx.getRowIndex(), 1); } // put the output value in the cache if (iout == tempValue) cachedValues.add(output, iout); } }
@Override public int compare(MatrixIndexes m1, MatrixIndexes m2) { return m1.compareTo(m2); }
/** * @param in * @param ixrange * @param brlen * @param bclen * @param rlen * @param clen * @param outlist * @throws DMLRuntimeException */ public static void performShift( IndexedMatrixValue in, IndexRange ixrange, int brlen, int bclen, long rlen, long clen, ArrayList<IndexedMatrixValue> outlist) throws DMLRuntimeException { MatrixIndexes ix = in.getIndexes(); MatrixBlock mb = (MatrixBlock) in.getValue(); long start_lhs_globalRowIndex = ixrange.rowStart + (ix.getRowIndex() - 1) * brlen; long start_lhs_globalColIndex = ixrange.colStart + (ix.getColumnIndex() - 1) * bclen; long end_lhs_globalRowIndex = start_lhs_globalRowIndex + mb.getNumRows() - 1; long end_lhs_globalColIndex = start_lhs_globalColIndex + mb.getNumColumns() - 1; long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen); long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen); long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen); long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen); for (long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) { for (long leftColIndex = start_lhs_colIndex; leftColIndex <= end_lhs_colIndex; leftColIndex++) { // Calculate global index of right hand side block long lhs_rl = Math.max((leftRowIndex - 1) * brlen + 1, start_lhs_globalRowIndex); long lhs_ru = Math.min(leftRowIndex * brlen, end_lhs_globalRowIndex); long lhs_cl = Math.max((leftColIndex - 1) * bclen + 1, start_lhs_globalColIndex); long lhs_cu = Math.min(leftColIndex * bclen, end_lhs_globalColIndex); int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen); int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen); int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen); int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen); long rhs_rl = lhs_rl - ixrange.rowStart + 1; long rhs_ru = rhs_rl + (lhs_ru - lhs_rl); long rhs_cl = lhs_cl - ixrange.colStart + 1; long rhs_cu = rhs_cl + (lhs_cu - lhs_cl); int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen); int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen); int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen); int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen); MatrixBlock slicedRHSBlk = mb.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock()); int lbrlen = UtilFunctions.computeBlockSize(rlen, leftRowIndex, brlen); int lbclen = UtilFunctions.computeBlockSize(clen, leftColIndex, bclen); MatrixBlock resultBlock = new MatrixBlock(lbrlen, lbclen, false); resultBlock = resultBlock.leftIndexingOperations( slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, UpdateType.COPY); outlist.add( new IndexedMatrixValue(new MatrixIndexes(leftRowIndex, leftColIndex), resultBlock)); } } }
@SuppressWarnings("deprecation") public void flushBuffer(Reporter reporter) throws RuntimeException { try { if (_mapBuffer != null) { MatrixIndexes key = null; // new MatrixIndexes(); MatrixCell value = new MatrixCell(); for (Entry<Byte, CTableMap> ctable : _mapBuffer.entrySet()) { ArrayList<Integer> resultIDs = ReduceBase.getOutputIndexes(ctable.getKey(), _resultIndexes); CTableMap resultMap = ctable.getValue(); // maintain result dims and nonzeros for (Integer i : resultIDs) { _resultNonZeros[i] += resultMap.size(); if (_resultDimsUnknown[i] == (byte) 1) { _resultMaxRowDims[i] = Math.max(resultMap.getMaxRow(), _resultMaxRowDims[i]); _resultMaxColDims[i] = Math.max(resultMap.getMaxColumn(), _resultMaxColDims[i]); } } // output result data for (LLDoubleEntry e : resultMap.entrySet()) { key = new MatrixIndexes(e.key1, e.key2); value.setValue(e.value); for (Integer i : resultIDs) { _collector.collectOutput(key, value, i, reporter); } } } } else if (_blockBuffer != null) { MatrixIndexes key = new MatrixIndexes(1, 1); // DataConverter.writeBinaryBlockMatrixToHDFS(path, job, mat, mc.get_rows(), mc.get_cols(), // mc.get_rows_per_block(), mc.get_cols_per_block(), replication); for (Entry<Byte, MatrixBlock> ctable : _blockBuffer.entrySet()) { ArrayList<Integer> resultIDs = ReduceBase.getOutputIndexes(ctable.getKey(), _resultIndexes); MatrixBlock outBlock = ctable.getValue(); outBlock.recomputeNonZeros(); // TODO: change hard coding of 1000 int brlen = 1000, bclen = 1000; int rlen = outBlock.getNumRows(); int clen = outBlock.getNumColumns(); // final output matrix is smaller than a single block if (rlen <= brlen && clen <= brlen) { key = new MatrixIndexes(1, 1); for (Integer i : resultIDs) { _collector.collectOutput(key, outBlock, i, reporter); _resultNonZeros[i] += outBlock.getNonZeros(); } } else { // Following code is similar to that in // DataConverter.DataConverter.writeBinaryBlockMatrixToHDFS // initialize blocks for reuse (at most 4 different blocks required) MatrixBlock[] blocks = MatrixWriter.createMatrixBlocksForReuse( rlen, clen, brlen, bclen, true, outBlock.getNonZeros()); // create and write subblocks of matrix for (int blockRow = 0; blockRow < (int) Math.ceil(rlen / (double) brlen); blockRow++) { for (int blockCol = 0; blockCol < (int) Math.ceil(clen / (double) bclen); blockCol++) { int maxRow = (blockRow * brlen + brlen < rlen) ? brlen : rlen - blockRow * brlen; int maxCol = (blockCol * bclen + bclen < clen) ? bclen : clen - blockCol * bclen; int row_offset = blockRow * brlen; int col_offset = blockCol * bclen; // get reuse matrix block MatrixBlock block = MatrixWriter.getMatrixBlockForReuse(blocks, maxRow, maxCol, brlen, bclen); // copy submatrix to block outBlock.sliceOperations( row_offset, row_offset + maxRow - 1, col_offset, col_offset + maxCol - 1, block); // TODO: skip empty "block" // append block to sequence file key.setIndexes(blockRow + 1, blockCol + 1); for (Integer i : resultIDs) { _collector.collectOutput(key, block, i, reporter); _resultNonZeros[i] += block.getNonZeros(); } // reset block for later reuse block.reset(); } } } } } else { throw new DMLRuntimeException("Unexpected.. both ctable buffers are empty."); } } catch (Exception ex) { throw new RuntimeException("Failed to flush ctable buffer.", ex); } // remove existing partial ctables if (_mapBuffer != null) _mapBuffer.clear(); else _blockBuffer.clear(); }