@Override
    public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
        throws Exception {
      MatrixIndexes ixIn = arg0._1();
      MatrixBlock blkIn = arg0._2();

      MatrixIndexes ixOut = new MatrixIndexes();
      MatrixBlock blkOut = new MatrixBlock();

      if (_type == CacheType.LEFT) {
        // get the right hand side matrix
        MatrixBlock left = _pbc.getMatrixBlock(1, (int) ixIn.getRowIndex());

        // execute matrix-vector mult
        OperationsOnMatrixValues.performAggregateBinary(
            new MatrixIndexes(1, ixIn.getRowIndex()), left, ixIn, blkIn, ixOut, blkOut, _op);
      } else // if( _type == CacheType.RIGHT )
      {
        // get the right hand side matrix
        MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), 1);

        // execute matrix-vector mult
        OperationsOnMatrixValues.performAggregateBinary(
            ixIn, blkIn, new MatrixIndexes(ixIn.getColumnIndex(), 1), right, ixOut, blkOut, _op);
      }

      // output new tuple
      return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut);
    }
  public static void performAggregateBinary(
      MatrixIndexes indexes1,
      MatrixValue value1,
      MatrixIndexes indexes2,
      MatrixValue value2,
      MatrixIndexes indexesOut,
      MatrixValue valueOut,
      AggregateBinaryOperator op)
      throws DMLRuntimeException {
    // compute output index
    indexesOut.setIndexes(indexes1.getRowIndex(), indexes2.getColumnIndex());

    // perform on the value
    value1.aggregateBinaryOperations(indexes1, value1, indexes2, value2, valueOut, op);
  }
Пример #3
0
  /**
   * @param ix
   * @param brlen
   * @param bclen
   * @param rl
   * @param ru
   * @param cl
   * @param cu
   * @return
   */
  public static boolean isInBlockRange(
      MatrixIndexes ix, int brlen, int bclen, long rl, long ru, long cl, long cu) {
    long bRLowerIndex = (ix.getRowIndex() - 1) * brlen + 1;
    long bRUpperIndex = ix.getRowIndex() * brlen;
    long bCLowerIndex = (ix.getColumnIndex() - 1) * bclen + 1;
    long bCUpperIndex = ix.getColumnIndex() * bclen;

    if (rl > bRUpperIndex || ru < bRLowerIndex) {
      return false;
    } else if (cl > bCUpperIndex || cu < bCLowerIndex) {
      return false;
    } else {
      return true;
    }
  }
Пример #4
0
    @Override
    public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(
        Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception {
      ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret =
          new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>();
      MatrixIndexes ixIn = arg0._1();
      MatrixBlock mb2 = arg0._2();

      // get the right hand side matrix
      MatrixBlock mb1 = _pmV.getMatrixBlock((int) ixIn.getRowIndex(), 1);

      // compute target block indexes
      long minPos = UtilFunctions.toLong(mb1.minNonZero());
      long maxPos = UtilFunctions.toLong(mb1.max());
      long rowIX1 = (minPos - 1) / _brlen + 1;
      long rowIX2 = (maxPos - 1) / _brlen + 1;
      boolean multipleOuts = (rowIX1 != rowIX2);

      if (minPos >= 1) // at least one row selected
      {
        // output sparsity estimate
        double spmb1 = OptimizerUtils.getSparsity(mb1.getNumRows(), 1, mb1.getNonZeros());
        long estnnz = (long) (spmb1 * mb2.getNonZeros());
        boolean sparse = MatrixBlock.evalSparseFormatInMemory(_brlen, mb2.getNumColumns(), estnnz);

        // compute and allocate output blocks
        MatrixBlock out1 = new MatrixBlock();
        MatrixBlock out2 = multipleOuts ? new MatrixBlock() : null;
        out1.reset(_brlen, mb2.getNumColumns(), sparse);
        if (out2 != null)
          out2.reset(
              UtilFunctions.computeBlockSize(_rlen, rowIX2, _brlen), mb2.getNumColumns(), sparse);

        // compute core matrix permutation (assumes that out1 has default blocksize,
        // hence we do a meta data correction afterwards)
        mb1.permutationMatrixMultOperations(mb2, out1, out2);
        out1.setNumRows(UtilFunctions.computeBlockSize(_rlen, rowIX1, _brlen));
        ret.add(
            new Tuple2<MatrixIndexes, MatrixBlock>(
                new MatrixIndexes(rowIX1, ixIn.getColumnIndex()), out1));
        if (out2 != null)
          ret.add(
              new Tuple2<MatrixIndexes, MatrixBlock>(
                  new MatrixIndexes(rowIX2, ixIn.getColumnIndex()), out2));
      }

      return ret;
    }
  @Override
  public void processInstruction(
      Class<? extends MatrixValue> valueClass,
      CachedValueMap cachedValues,
      IndexedMatrixValue tempValue,
      IndexedMatrixValue zeroInput,
      int blockRowFactor,
      int blockColFactor)
      throws DMLUnsupportedOperationException, DMLRuntimeException {

    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
    if (blkList != null)
      for (IndexedMatrixValue in : blkList) {
        if (in == null) continue;

        // allocate space for the output value
        IndexedMatrixValue out;
        if (input == output) out = tempValue;
        else out = cachedValues.holdPlace(output, valueClass);

        MatrixIndexes inix = in.getIndexes();

        // prune unnecessary blocks for trace
        if ((((AggregateUnaryOperator) optr).indexFn instanceof ReduceDiag
            && inix.getColumnIndex() != inix.getRowIndex())) {
          // do nothing (block not on diagonal); but reset
          out.getValue().reset();
        } else // general case
        {
          // process instruction
          AggregateUnaryOperator auop = (AggregateUnaryOperator) optr;
          OperationsOnMatrixValues.performAggregateUnary(
              inix,
              in.getValue(),
              out.getIndexes(),
              out.getValue(),
              auop,
              blockRowFactor,
              blockColFactor);
          if (_dropCorr)
            ((MatrixBlock) out.getValue()).dropLastRowsOrColums(auop.aggOp.correctionLocation);
        }

        // put the output value in the cache
        if (out == tempValue) cachedValues.add(output, out);
      }
  }
    @Override
    public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
        throws Exception {
      MatrixBlock pmV = _pmV.getMatrixBlock(1, 1);

      MatrixIndexes ixIn = arg0._1();
      MatrixBlock blkIn = arg0._2();
      int rowIx = (int) ixIn.getRowIndex();

      MatrixIndexes ixOut = new MatrixIndexes(1, 1);
      MatrixBlock blkOut = new MatrixBlock();

      // execute mapmmchain operation
      blkIn.chainMatrixMultOperations(pmV, _pmW.getMatrixBlock(rowIx, 1), blkOut, ChainType.XtwXv);

      // output new tuple
      return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut);
    }
    @Override
    public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(
        Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception {
      ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret =
          new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>();

      MatrixIndexes ixIn = arg0._1();
      MatrixBlock blkIn = arg0._2();

      if (_type == CacheType.LEFT) {
        // for all matching left-hand-side blocks
        int len = _pbc.getNumRowBlocks();
        for (int i = 1; i <= len; i++) {
          MatrixBlock left = _pbc.getMatrixBlock(i, (int) ixIn.getRowIndex());
          MatrixIndexes ixOut = new MatrixIndexes();
          MatrixBlock blkOut = new MatrixBlock();

          // execute matrix-vector mult
          OperationsOnMatrixValues.performAggregateBinary(
              new MatrixIndexes(i, ixIn.getRowIndex()), left, ixIn, blkIn, ixOut, blkOut, _op);

          ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut));
        }
      } else // if( _type == CacheType.RIGHT )
      {
        // for all matching right-hand-side blocks
        int len = _pbc.getNumColumnBlocks();
        for (int j = 1; j <= len; j++) {
          // get the right hand side matrix
          MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), j);
          MatrixIndexes ixOut = new MatrixIndexes();
          MatrixBlock blkOut = new MatrixBlock();

          // execute matrix-vector mult
          OperationsOnMatrixValues.performAggregateBinary(
              ixIn, blkIn, new MatrixIndexes(ixIn.getColumnIndex(), j), right, ixOut, blkOut, _op);

          ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut));
        }
      }

      return ret;
    }
 public static void performZeroOut(
     MatrixIndexes indexesIn,
     MatrixValue valueIn,
     MatrixIndexes indexesOut,
     MatrixValue valueOut,
     IndexRange range,
     boolean complementary)
     throws DMLRuntimeException {
   valueIn.zeroOutOperations(valueOut, range, complementary);
   indexesOut.setIndexes(indexesIn);
 }
  /**
   * @param target
   * @param groups
   * @param brlen
   * @param bclen
   * @param outlist
   * @throws DMLRuntimeException
   */
  public static void performMapGroupedAggregate(
      Operator op,
      IndexedMatrixValue inTarget,
      MatrixBlock groups,
      int ngroups,
      int brlen,
      int bclen,
      ArrayList<IndexedMatrixValue> outlist)
      throws DMLRuntimeException {
    MatrixIndexes ix = inTarget.getIndexes();
    MatrixBlock target = (MatrixBlock) inTarget.getValue();

    // execute grouped aggregate operations
    MatrixBlock out = groups.groupedAggOperations(target, null, new MatrixBlock(), ngroups, op);

    if (out.getNumRows() <= brlen && out.getNumColumns() <= bclen) {
      // single output block
      outlist.add(new IndexedMatrixValue(new MatrixIndexes(1, ix.getColumnIndex()), out));
    } else {
      // multiple output blocks (by op def, single column block )
      for (int blockRow = 0;
          blockRow < (int) Math.ceil(out.getNumRows() / (double) brlen);
          blockRow++) {
        int maxRow =
            (blockRow * brlen + brlen < out.getNumRows())
                ? brlen
                : out.getNumRows() - blockRow * brlen;
        int row_offset = blockRow * brlen;

        // copy submatrix to block
        MatrixBlock tmp =
            out.sliceOperations(
                row_offset, row_offset + maxRow - 1, 0, out.getNumColumns() - 1, new MatrixBlock());

        // append block to result cache
        outlist.add(
            new IndexedMatrixValue(new MatrixIndexes(blockRow + 1, ix.getColumnIndex()), tmp));
      }
    }
  }
    @Override
    public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
        throws Exception {
      MatrixIndexes ixIn = arg0._1();
      MatrixBlock blkIn = arg0._2();

      MatrixIndexes ixOut = new MatrixIndexes();
      MatrixBlock blkOut = new MatrixBlock();

      // process instruction
      OperationsOnMatrixValues.performAggregateUnary(
          ixIn, blkIn, ixOut, blkOut, ((AggregateUnaryOperator) _op), _brlen, _bclen);
      if (((AggregateUnaryOperator) _op).aggOp.correctionExists)
        blkOut.dropLastRowsOrColums(((AggregateUnaryOperator) _op).aggOp.correctionLocation);

      // cumsum expand partial aggregates
      long rlenOut = (long) Math.ceil((double) _rlen / _brlen);
      long rixOut = (long) Math.ceil((double) ixIn.getRowIndex() / _brlen);
      int rlenBlk = (int) Math.min(rlenOut - (rixOut - 1) * _brlen, _brlen);
      int clenBlk = blkOut.getNumColumns();
      int posBlk = (int) ((ixIn.getRowIndex() - 1) % _brlen);
      MatrixBlock blkOut2 = new MatrixBlock(rlenBlk, clenBlk, false);
      blkOut2.copy(posBlk, posBlk, 0, clenBlk - 1, blkOut, true);
      ixOut.setIndexes(rixOut, ixOut.getColumnIndex());

      // output new tuple
      return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut2);
    }
Пример #11
0
      @Override
      protected Tuple2<MatrixIndexes, MatrixBlock> computeNext(
          Tuple2<MatrixIndexes, MatrixBlock> arg) throws Exception {
        MatrixIndexes ixIn = arg._1();
        MatrixBlock blkIn = arg._2();
        MatrixBlock blkOut = new MatrixBlock();

        if (_type == CacheType.LEFT) {
          // get the right hand side matrix
          MatrixBlock left = _pbc.getMatrixBlock(1, (int) ixIn.getRowIndex());

          // execute index preserving matrix multiplication
          left.aggregateBinaryOperations(left, blkIn, blkOut, _op);
        } else // if( _type == CacheType.RIGHT )
        {
          // get the right hand side matrix
          MatrixBlock right = _pbc.getMatrixBlock((int) ixIn.getColumnIndex(), 1);

          // execute index preserving matrix multiplication
          blkIn.aggregateBinaryOperations(blkIn, right, blkOut, _op);
        }

        return new Tuple2<MatrixIndexes, MatrixBlock>(ixIn, blkOut);
      }
  @Override
  public void processInstruction(
      Class<? extends MatrixValue> valueClass,
      CachedValueMap cachedValues,
      IndexedMatrixValue tempValue,
      IndexedMatrixValue zeroInput,
      int blockRowFactor,
      int blockColFactor)
      throws DMLRuntimeException {
    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
    if (blkList == null) return;

    for (IndexedMatrixValue in1 : blkList) {
      if (in1 == null) continue;

      MatrixIndexes inix = in1.getIndexes();
      MatrixBlock blk = (MatrixBlock) in1.getValue();
      long rixOffset = (inix.getRowIndex() - 1) * blockRowFactor;
      boolean firstBlk = (inix.getRowIndex() == 1);
      boolean lastBlk = (inix.getRowIndex() == _lastRowBlockIndex);

      // introduce offsets w/ init value for first row
      if (firstBlk) {
        IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
        ((MatrixBlock) out.getValue()).reset(1, blk.getNumColumns());
        if (_initValue != 0) {
          for (int j = 0; j < blk.getNumColumns(); j++)
            ((MatrixBlock) out.getValue()).appendValue(0, j, _initValue);
        }
        out.getIndexes().setIndexes(1, inix.getColumnIndex());
      }

      // output splitting (shift by one), preaggregated offset used by subsequent block
      for (int i = 0; i < blk.getNumRows(); i++)
        if (!(lastBlk && i == (blk.getNumRows() - 1))) // ignore last row
        {
          IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
          MatrixBlock tmpBlk = (MatrixBlock) out.getValue();
          tmpBlk.reset(1, blk.getNumColumns());
          blk.sliceOperations(i, i, 0, blk.getNumColumns() - 1, tmpBlk);
          out.getIndexes().setIndexes(rixOffset + i + 2, inix.getColumnIndex());
        }
    }
  }
  /**
   * @param path
   * @param job
   * @param fs
   * @param dest
   * @param rlen
   * @param clen
   * @param brlen
   * @param bclen
   * @throws IOException
   */
  @SuppressWarnings("deprecation")
  private void readBinaryCellMatrixFromHDFS(
      Path path,
      JobConf job,
      FileSystem fs,
      MatrixBlock dest,
      long rlen,
      long clen,
      int brlen,
      int bclen)
      throws IOException {
    boolean sparse = dest.isInSparseFormat();
    MatrixIndexes key = new MatrixIndexes();
    MatrixCell value = new MatrixCell();
    int row = -1;
    int col = -1;

    try {
      for (Path lpath : getSequenceFilePaths(fs, path)) // 1..N files
      {
        // directly read from sequence files (individual partfiles)
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, lpath, job);

        try {
          if (sparse) {
            while (reader.next(key, value)) {
              row = (int) key.getRowIndex() - 1;
              col = (int) key.getColumnIndex() - 1;
              double lvalue = value.getValue();
              dest.appendValue(row, col, lvalue);
            }
          } else {
            while (reader.next(key, value)) {
              row = (int) key.getRowIndex() - 1;
              col = (int) key.getColumnIndex() - 1;
              double lvalue = value.getValue();
              dest.appendValue(row, col, lvalue);
            }
          }
        } finally {
          IOUtilFunctions.closeSilently(reader);
        }
      }

      if (sparse) dest.sortSparseRows();
    } catch (Exception ex) {
      // post-mortem error handling and bounds checking
      if (row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen) {
        throw new IOException(
            "Matrix cell ["
                + (row + 1)
                + ","
                + (col + 1)
                + "] "
                + "out of overall matrix range [1:"
                + rlen
                + ",1:"
                + clen
                + "].");
      } else {
        throw new IOException("Unable to read matrix in binary cell format.", ex);
      }
    }
  }
  @Override
  public void processInstruction(
      Class<? extends MatrixValue> valueClass,
      CachedValueMap cachedValues,
      IndexedMatrixValue tempValue,
      IndexedMatrixValue zeroInput,
      int blockRowFactor,
      int blockColFactor)
      throws DMLRuntimeException {
    QuaternaryOperator qop = (QuaternaryOperator) optr;

    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(_input1);
    if (blkList != null)
      for (IndexedMatrixValue imv : blkList) {
        // Step 1: prepare inputs and output
        if (imv == null) continue;
        MatrixIndexes inIx = imv.getIndexes();
        MatrixValue inVal = imv.getValue();

        // allocate space for the output value
        IndexedMatrixValue iout = null;
        if (output == _input1) iout = tempValue;
        else iout = cachedValues.holdPlace(output, valueClass);

        MatrixIndexes outIx = iout.getIndexes();
        MatrixValue outVal = iout.getValue();

        // Step 2: get remaining inputs: Wij, Ui, Vj
        MatrixValue Xij = inVal;

        // get Wij if existing (null of WeightsType.NONE or WSigmoid any type)
        IndexedMatrixValue iWij = (_input4 != -1) ? cachedValues.getFirst(_input4) : null;
        MatrixValue Wij = (iWij != null) ? iWij.getValue() : null;
        if (null == Wij && qop.hasFourInputs()) {
          MatrixBlock mb = new MatrixBlock(1, 1, false);
          String[] parts = InstructionUtils.getInstructionParts(instString);
          mb.quickSetValue(0, 0, Double.valueOf(parts[4]));
          Wij = mb;
        }

        // get Ui and Vj, potentially through distributed cache
        MatrixValue Ui =
            (!_cacheU)
                ? cachedValues.getFirst(_input2).getValue() // U
                : MRBaseForCommonInstructions.dcValues
                    .get(_input2)
                    .getDataBlock((int) inIx.getRowIndex(), 1)
                    .getValue();
        MatrixValue Vj =
            (!_cacheV)
                ? cachedValues.getFirst(_input3).getValue() // t(V)
                : MRBaseForCommonInstructions.dcValues
                    .get(_input3)
                    .getDataBlock((int) inIx.getColumnIndex(), 1)
                    .getValue();
        // handle special input case: //V through shuffle -> t(V)
        if (Ui.getNumColumns() != Vj.getNumColumns()) {
          Vj =
              LibMatrixReorg.reorg(
                  (MatrixBlock) Vj,
                  new MatrixBlock(Vj.getNumColumns(), Vj.getNumRows(), Vj.isInSparseFormat()),
                  new ReorgOperator(SwapIndex.getSwapIndexFnObject()));
        }

        // Step 3: process instruction
        Xij.quaternaryOperations(qop, Ui, Vj, Wij, outVal);

        // set output indexes

        if (qop.wtype1 != null || qop.wtype4 != null) outIx.setIndexes(1, 1); // wsloss
        else if (qop.wtype2 != null
            || qop.wtype5 != null
            || qop.wtype3 != null && qop.wtype3.isBasic())
          outIx.setIndexes(inIx); // wsigmoid/wdivmm-basic
        else { // wdivmm
          boolean left = qop.wtype3.isLeft();
          outIx.setIndexes(left ? inIx.getColumnIndex() : inIx.getRowIndex(), 1);
        }

        // put the output value in the cache
        if (iout == tempValue) cachedValues.add(output, iout);
      }
  }
 @Override
 public int compare(MatrixIndexes m1, MatrixIndexes m2) {
   return m1.compareTo(m2);
 }
  /**
   * @param in
   * @param ixrange
   * @param brlen
   * @param bclen
   * @param rlen
   * @param clen
   * @param outlist
   * @throws DMLRuntimeException
   */
  public static void performShift(
      IndexedMatrixValue in,
      IndexRange ixrange,
      int brlen,
      int bclen,
      long rlen,
      long clen,
      ArrayList<IndexedMatrixValue> outlist)
      throws DMLRuntimeException {
    MatrixIndexes ix = in.getIndexes();
    MatrixBlock mb = (MatrixBlock) in.getValue();

    long start_lhs_globalRowIndex = ixrange.rowStart + (ix.getRowIndex() - 1) * brlen;
    long start_lhs_globalColIndex = ixrange.colStart + (ix.getColumnIndex() - 1) * bclen;
    long end_lhs_globalRowIndex = start_lhs_globalRowIndex + mb.getNumRows() - 1;
    long end_lhs_globalColIndex = start_lhs_globalColIndex + mb.getNumColumns() - 1;

    long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen);
    long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen);
    long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen);
    long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen);

    for (long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
      for (long leftColIndex = start_lhs_colIndex;
          leftColIndex <= end_lhs_colIndex;
          leftColIndex++) {

        // Calculate global index of right hand side block
        long lhs_rl = Math.max((leftRowIndex - 1) * brlen + 1, start_lhs_globalRowIndex);
        long lhs_ru = Math.min(leftRowIndex * brlen, end_lhs_globalRowIndex);
        long lhs_cl = Math.max((leftColIndex - 1) * bclen + 1, start_lhs_globalColIndex);
        long lhs_cu = Math.min(leftColIndex * bclen, end_lhs_globalColIndex);

        int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen);
        int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen);
        int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen);
        int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen);

        long rhs_rl = lhs_rl - ixrange.rowStart + 1;
        long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
        long rhs_cl = lhs_cl - ixrange.colStart + 1;
        long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);

        int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen);
        int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen);
        int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen);
        int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen);

        MatrixBlock slicedRHSBlk =
            mb.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock());

        int lbrlen = UtilFunctions.computeBlockSize(rlen, leftRowIndex, brlen);
        int lbclen = UtilFunctions.computeBlockSize(clen, leftColIndex, bclen);
        MatrixBlock resultBlock = new MatrixBlock(lbrlen, lbclen, false);
        resultBlock =
            resultBlock.leftIndexingOperations(
                slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, UpdateType.COPY);
        outlist.add(
            new IndexedMatrixValue(new MatrixIndexes(leftRowIndex, leftColIndex), resultBlock));
      }
    }
  }
Пример #17
0
  @SuppressWarnings("deprecation")
  public void flushBuffer(Reporter reporter) throws RuntimeException {
    try {
      if (_mapBuffer != null) {
        MatrixIndexes key = null; // new MatrixIndexes();
        MatrixCell value = new MatrixCell();
        for (Entry<Byte, CTableMap> ctable : _mapBuffer.entrySet()) {
          ArrayList<Integer> resultIDs =
              ReduceBase.getOutputIndexes(ctable.getKey(), _resultIndexes);
          CTableMap resultMap = ctable.getValue();

          // maintain result dims and nonzeros
          for (Integer i : resultIDs) {
            _resultNonZeros[i] += resultMap.size();
            if (_resultDimsUnknown[i] == (byte) 1) {
              _resultMaxRowDims[i] = Math.max(resultMap.getMaxRow(), _resultMaxRowDims[i]);
              _resultMaxColDims[i] = Math.max(resultMap.getMaxColumn(), _resultMaxColDims[i]);
            }
          }

          // output result data
          for (LLDoubleEntry e : resultMap.entrySet()) {
            key = new MatrixIndexes(e.key1, e.key2);
            value.setValue(e.value);
            for (Integer i : resultIDs) {
              _collector.collectOutput(key, value, i, reporter);
            }
          }
        }
      } else if (_blockBuffer != null) {
        MatrixIndexes key = new MatrixIndexes(1, 1);
        // DataConverter.writeBinaryBlockMatrixToHDFS(path, job, mat, mc.get_rows(), mc.get_cols(),
        // mc.get_rows_per_block(), mc.get_cols_per_block(), replication);
        for (Entry<Byte, MatrixBlock> ctable : _blockBuffer.entrySet()) {
          ArrayList<Integer> resultIDs =
              ReduceBase.getOutputIndexes(ctable.getKey(), _resultIndexes);
          MatrixBlock outBlock = ctable.getValue();
          outBlock.recomputeNonZeros();

          // TODO: change hard coding of 1000
          int brlen = 1000, bclen = 1000;
          int rlen = outBlock.getNumRows();
          int clen = outBlock.getNumColumns();

          // final output matrix is smaller than a single block
          if (rlen <= brlen && clen <= brlen) {
            key = new MatrixIndexes(1, 1);
            for (Integer i : resultIDs) {
              _collector.collectOutput(key, outBlock, i, reporter);
              _resultNonZeros[i] += outBlock.getNonZeros();
            }
          } else {
            // Following code is similar to that in
            // DataConverter.DataConverter.writeBinaryBlockMatrixToHDFS
            // initialize blocks for reuse (at most 4 different blocks required)
            MatrixBlock[] blocks =
                MatrixWriter.createMatrixBlocksForReuse(
                    rlen, clen, brlen, bclen, true, outBlock.getNonZeros());

            // create and write subblocks of matrix
            for (int blockRow = 0; blockRow < (int) Math.ceil(rlen / (double) brlen); blockRow++) {
              for (int blockCol = 0;
                  blockCol < (int) Math.ceil(clen / (double) bclen);
                  blockCol++) {
                int maxRow = (blockRow * brlen + brlen < rlen) ? brlen : rlen - blockRow * brlen;
                int maxCol = (blockCol * bclen + bclen < clen) ? bclen : clen - blockCol * bclen;

                int row_offset = blockRow * brlen;
                int col_offset = blockCol * bclen;

                // get reuse matrix block
                MatrixBlock block =
                    MatrixWriter.getMatrixBlockForReuse(blocks, maxRow, maxCol, brlen, bclen);

                // copy submatrix to block
                outBlock.sliceOperations(
                    row_offset,
                    row_offset + maxRow - 1,
                    col_offset,
                    col_offset + maxCol - 1,
                    block);

                // TODO: skip empty "block"

                // append block to sequence file
                key.setIndexes(blockRow + 1, blockCol + 1);
                for (Integer i : resultIDs) {
                  _collector.collectOutput(key, block, i, reporter);
                  _resultNonZeros[i] += block.getNonZeros();
                }

                // reset block for later reuse
                block.reset();
              }
            }
          }
        }
      } else {
        throw new DMLRuntimeException("Unexpected.. both ctable buffers are empty.");
      }
    } catch (Exception ex) {
      throw new RuntimeException("Failed to flush ctable buffer.", ex);
    }
    // remove existing partial ctables
    if (_mapBuffer != null) _mapBuffer.clear();
    else _blockBuffer.clear();
  }