// output the records in the outCache. public void close() throws IOException { long start = System.currentTimeMillis(); Iterator<Entry<MatrixIndexes, MatrixValue>> it = outCache.entrySet().iterator(); while (it.hasNext()) { Entry<MatrixIndexes, MatrixValue> entry = it.next(); realWriteToCollector(entry.getKey(), entry.getValue()); } // handle empty block output (on first reduce task only) if (outputDummyRecords) // required for rejecting empty blocks in mappers { long rlen = dim1.getRows(); long clen = dim2.getCols(); int brlen = dim1.getRowsPerBlock(); int bclen = dim2.getColsPerBlock(); MatrixIndexes tmpIx = new MatrixIndexes(); MatrixBlock tmpVal = new MatrixBlock(); for (long i = 0, r = 1; i < rlen; i += brlen, r++) for (long j = 0, c = 1; j < clen; j += bclen, c++) { int realBrlen = (int) Math.min((long) brlen, rlen - (r - 1) * brlen); int realBclen = (int) Math.min((long) bclen, clen - (c - 1) * bclen); tmpIx.setIndexes(r, c); tmpVal.reset(realBrlen, realBclen); collectFinalMultipleOutputs.collectOutput(tmpIx, tmpVal, 0, cachedReporter); } } if (cachedReporter != null) cachedReporter.incrCounter( Counters.COMBINE_OR_REDUCE_TIME, System.currentTimeMillis() - start); super.close(); }
private void processTernaryCombineInstruction(CombineTernaryInstruction ins, Reporter reporter) throws IOException { IndexedMatrixValue in1 = cachedValues.getFirst(ins.input1); IndexedMatrixValue in2 = cachedValues.getFirst(ins.input2); IndexedMatrixValue in3 = cachedValues.getFirst(ins.input3); if (in1 == null && in2 == null && in3 == null) return; int nr = 0, nc = 0; if (in1 != null) { nr = in1.getValue().getNumRows(); nc = in1.getValue().getNumColumns(); } else if (in2 != null) { nr = in2.getValue().getNumRows(); nc = in2.getValue().getNumColumns(); } else { nr = in3.getValue().getNumRows(); nc = in3.getValue().getNumColumns(); } // if one of the inputs is null, then it is a all zero block if (in1 == null) { in1 = zeroInput; in1.getValue().reset(nr, nc); } if (in2 == null) { in2 = zeroInput; in2.getValue().reset(nr, nc); } if (in3 == null) { in3 = zeroInput; in3.getValue().reset(nr, nc); } // process instruction try { ArrayList<Integer> outputIndexes = outputIndexesMapping.get(ins.output); for (int r = 0; r < nr; r++) for (int c = 0; c < nc; c++) { Pair<Integer, Integer> blockSize = outputBlockSizes.get(ins.output); keyBuff.setIndexes( UtilFunctions.cellIndexCalculation( in1.getIndexes().getRowIndex(), blockSize.getKey(), r), UtilFunctions.cellIndexCalculation( in1.getIndexes().getColumnIndex(), blockSize.getValue(), c)); valueBuff.setValue(in1.getValue().getValue(r, c)); valueBuff.setOtherValue(in2.getValue().getValue(r, c)); valueBuff.setWeight(in3.getValue().getValue(r, c)); for (int i : outputIndexes) { collectFinalMultipleOutputs.collectOutput(keyBuff, valueBuff, i, reporter); // System.out.println("output: "+keyBuff+" -- "+valueBuff); } } } catch (Exception e) { throw new RuntimeException(e); } }
private void processBinaryCombineInstruction(CombineBinaryInstruction ins, Reporter reporter) throws IOException { IndexedMatrixValue in1 = cachedValues.getFirst(ins.input1); IndexedMatrixValue in2 = cachedValues.getFirst(ins.input2); if (in1 == null && in2 == null) return; MatrixIndexes indexes; if (in1 != null) indexes = in1.getIndexes(); else indexes = in2.getIndexes(); // if one of the inputs is null, then it is a all zero block if (in1 == null) { in1 = zeroInput; in1.getValue().reset(in2.getValue().getNumRows(), in2.getValue().getNumColumns()); } if (in2 == null) { in2 = zeroInput; in2.getValue().reset(in1.getValue().getNumRows(), in1.getValue().getNumColumns()); } // System.out.println("in1:"+in1); // System.out.println("in2:"+in2); // process instruction try { /*in1.getValue().combineOperations(in2.getValue(), collectFinalMultipleOutputs, reporter, keyBuff, valueBuff, getOutputIndexes(ins.output));*/ ArrayList<Integer> outputIndexes = outputIndexesMapping.get(ins.output); for (int r = 0; r < in1.getValue().getNumRows(); r++) for (int c = 0; c < in1.getValue().getNumColumns(); c++) { Pair<Integer, Integer> blockSize = outputBlockSizes.get(ins.output); keyBuff.setIndexes( UtilFunctions.cellIndexCalculation(indexes.getRowIndex(), blockSize.getKey(), r), UtilFunctions.cellIndexCalculation( indexes.getColumnIndex(), blockSize.getValue(), c)); valueBuff.setValue(in1.getValue().getValue(r, c)); double temp = in2.getValue().getValue(r, c); if (ins.isSecondInputWeight()) { valueBuff.setWeight(temp); valueBuff.setOtherValue(0); } else { valueBuff.setWeight(1); valueBuff.setOtherValue(temp); } for (int i : outputIndexes) { collectFinalMultipleOutputs.collectOutput(keyBuff, valueBuff, i, reporter); // System.out.println("output: "+keyBuff+" -- "+valueBuff); } } } catch (Exception e) { throw new RuntimeException(e); } }
private void processJoin(int tag, RemainIndexValue rValue) throws Exception { // for the cached matrix if (tag == 0) { addToCache(rValue, tag); // LOG.info("put in the buffer for left matrix"); // LOG.info(rblock.block.toString()); } else // for the probing matrix { // LOG.info("process join with block size: "+rValue.value.getNumRows()+" X // "+rValue.value.getNumColumns()+" nonZeros: "+rValue.value.getNonZeros()); for (int i = 0; i < cacheSize; i++) { RemainIndexValue left, right; if (tagForLeft == 0) { left = cache.get(i); right = rValue; } else { right = cache.get(i); left = rValue; } indexesbuffer.setIndexes(left.remainIndex, right.remainIndex); try { OperationsOnMatrixValues.performAggregateBinaryIgnoreIndexes( left.value, right.value, valueBuffer, (AggregateBinaryOperator) aggBinInstruction.getOperator()); } catch (DMLUnsupportedOperationException e) { throw new IOException(e); } // if(valueBuffer.getNonZeros()>0) collectOutput(indexesbuffer, valueBuffer); } } }
@Override public void execute(MatrixIndexes in, MatrixIndexes out) { // only used for V2M out.setIndexes(in.getRowIndex(), in.getRowIndex()); }
@Override public void execute(MatrixIndexes in, MatrixIndexes out) { out.setIndexes(1, 1); }