@Override public void map(Chunk cs[]) { _xsum = new double[_ncolx]; _ysum = new double[_ncoly]; double[] xvals = new double[_ncolx]; double[] yvals = new double[_ncoly]; double xval, yval; boolean add; int len = cs[0]._len; for (int row = 0; row < len; row++) { add = true; // reset existing arrays to 0 rather than initializing new ones to save on garbage // collection Arrays.fill(xvals, 0); Arrays.fill(yvals, 0); for (int y = 0; y < _ncoly; y++) { final Chunk cy = cs[y]; yval = cy.atd(row); // if any yval along a row is NA, discard the entire row if (Double.isNaN(yval)) { _NACount++; add = false; break; } yvals[y] = yval; } if (add) { for (int x = 0; x < _ncolx; x++) { final Chunk cx = cs[x + _ncoly]; xval = cx.atd(row); // if any xval along a row is NA, discard the entire row if (Double.isNaN(xval)) { _NACount++; add = false; break; } xvals[x] = xval; } } // add is true iff row has been traversed and found no NAs among yvals and xvals if (add) { ArrayUtils.add(_xsum, xvals); ArrayUtils.add(_ysum, yvals); } } }
@Override public void map(Chunk cs[]) { int ncoly = cs.length; _ysum = new double[ncoly]; double[] yvals = new double[ncoly]; double yval; boolean add; int len = cs[0]._len; for (int row = 0; row < len; row++) { add = true; Arrays.fill(yvals, 0); for (int y = 0; y < ncoly; y++) { final Chunk cy = cs[y]; yval = cy.atd(row); // if any yval along a row is NA, discard the entire row if (Double.isNaN(yval)) { _NACount++; add = false; break; } yvals[y] = yval; } if (add) { ArrayUtils.add(_ysum, yvals); } } }
@Override public void reduce(Histo h) { for (int i = 0; i < _nbins; i++) { // Keep min/max if (_mins[i] > h._mins[i]) _mins[i] = h._mins[i]; if (_maxs[i] < h._maxs[i]) _maxs[i] = h._maxs[i]; } ArrayUtils.add(_bins, h._bins); }
@Override public void reduce(Lloyds mr) { for (int clu = 0; clu < _k; clu++) { long ra = _size[clu]; long rb = mr._size[clu]; double[] ma = _cMeans[clu]; double[] mb = mr._cMeans[clu]; for (int c = 0; c < ma.length; c++) // Recursive mean if (ra + rb > 0) ma[c] = (ma[c] * ra + mb[c] * rb) / (ra + rb); } ArrayUtils.add(_cats, mr._cats); ArrayUtils.add(_cSqr, mr._cSqr); ArrayUtils.add(_size, mr._size); // track global worst-row if (_worst_err < mr._worst_err) { _worst_err = mr._worst_err; _worst_row = mr._worst_row; } }
/** * Add another model info into this This will add the weights/biases/learning rate helpers, and * the number of processed training samples Note: It will NOT add the elastic averaging helpers, * which are always kept constant (they already are the result of a reduction) * * @param other */ public void add(DeepLearningModelInfo other) { for (int i = 0; i < dense_row_weights.length; ++i) ArrayUtils.add(get_weights(i).raw(), other.get_weights(i).raw()); for (int i = 0; i < biases.length; ++i) ArrayUtils.add(biases[i].raw(), other.biases[i].raw()); if (avg_activations != null) for (int i = 0; i < avg_activations.length; ++i) ArrayUtils.add(avg_activations[i].raw(), other.biases[i].raw()); if (has_momenta()) { assert (other.has_momenta()); for (int i = 0; i < dense_row_weights_momenta.length; ++i) ArrayUtils.add(get_weights_momenta(i).raw(), other.get_weights_momenta(i).raw()); for (int i = 0; i < biases_momenta.length; ++i) ArrayUtils.add(biases_momenta[i].raw(), other.biases_momenta[i].raw()); } if (adaDelta()) { assert (other.adaDelta()); for (int i = 0; i < dense_row_ada_dx_g.length; ++i) { ArrayUtils.add(get_ada_dx_g(i).raw(), other.get_ada_dx_g(i).raw()); } } add_processed_local(other.get_processed_local()); }
public void reduce(MRTaskAtx other) { ArrayUtils.add(atx, other.atx); }
@Override public void reduce(AstCumu.CumuTask t) { if (_chkCumu != t._chkCumu) ArrayUtils.add(_chkCumu, t._chkCumu); }
@Override public void reduce(CoVarTaskCompleteObsSym cvt) { ArrayUtils.add(_covs, cvt._covs); }
@Override public void reduce(CoVarTaskCompleteObsMeanSym cvt) { ArrayUtils.add(_ysum, cvt._ysum); _NACount += cvt._NACount; }
@Override public void reduce(CoVarTaskEverything cvt) { ArrayUtils.add(_covs, cvt._covs); }
@Override public void reduce(CoxPHTask that) { n += that.n; sumWeights += that.sumWeights; ArrayUtils.add(sumWeightedCatX, that.sumWeightedCatX); ArrayUtils.add(sumWeightedNumX, that.sumWeightedNumX); ArrayUtils.add(sizeRiskSet, that.sizeRiskSet); ArrayUtils.add(sizeCensored, that.sizeCensored); ArrayUtils.add(sizeEvents, that.sizeEvents); ArrayUtils.add(countEvents, that.countEvents); ArrayUtils.add(sumXEvents, that.sumXEvents); ArrayUtils.add(sumRiskEvents, that.sumRiskEvents); ArrayUtils.add(sumXRiskEvents, that.sumXRiskEvents); ArrayUtils.add(sumXXRiskEvents, that.sumXXRiskEvents); ArrayUtils.add(sumLogRiskEvents, that.sumLogRiskEvents); ArrayUtils.add(rcumsumRisk, that.rcumsumRisk); ArrayUtils.add(rcumsumXRisk, that.rcumsumXRisk); ArrayUtils.add(rcumsumXXRisk, that.rcumsumXXRisk); }