private void rebalanceChunk(Vec srcVec, Chunk chk) { NewChunk dst = new NewChunk(chk); dst._len = dst._len2 = 0; int rem = chk._len; while (rem > 0 && dst._len2 < chk._len) { Chunk srcRaw = srcVec.chunkForRow(chk._start + dst._len2); NewChunk src = new NewChunk((srcRaw)); src = srcRaw.inflate_impl(src); assert src._len2 == srcRaw._len; int srcFrom = (int) (chk._start + dst._len2 - src._start); // check if the result is sparse (not exact since we only take subset of src in general) if ((src.sparse() && dst.sparse()) || (src._len + dst._len < NewChunk.MIN_SPARSE_RATIO * (src._len2 + dst._len2))) { src.set_sparse(src._len); dst.set_sparse(dst._len); } final int srcTo = srcFrom + rem; int off = srcFrom - 1; Iterator<NewChunk.Value> it = src.values(Math.max(0, srcFrom), srcTo); while (it.hasNext()) { NewChunk.Value v = it.next(); final int rid = v.rowId0(); assert rid < srcTo; int add = rid - off; off = rid; dst.addZeros(add - 1); v.add2Chunk(dst); rem -= add; assert rem >= 0; } int trailingZeros = Math.min(rem, src._len2 - off - 1); dst.addZeros(trailingZeros); rem -= trailingZeros; } assert rem == 0 : "rem = " + rem; assert dst._len2 == chk._len : "len2 = " + dst._len2 + ", _len = " + chk._len; dst.close(dst.cidx(), _fs); }
// Append all of 'nc' onto the current NewChunk. Kill nc. public void add(NewChunk nc) { assert _cidx >= 0; assert sparseLen() <= _len; assert nc.sparseLen() <= nc._len : "_len = " + nc.sparseLen() + ", _len2 = " + nc._len; if (nc._len == 0) return; if (_len == 0) { _ls = nc._ls; nc._ls = null; _xs = nc._xs; nc._xs = null; _id = nc._id; nc._id = null; _ds = nc._ds; nc._ds = null; _is = nc._is; nc._is = null; _ss = nc._ss; nc._ss = null; set_sparseLen(nc.sparseLen()); set_len(nc._len); return; } if (nc.sparse() != sparse()) { // for now, just make it dense cancel_sparse(); nc.cancel_sparse(); } if (_ds != null) throw H2O.fail(); while (sparseLen() + nc.sparseLen() >= _xs.length) _xs = MemoryManager.arrayCopyOf(_xs, _xs.length << 1); _ls = MemoryManager.arrayCopyOf(_ls, _xs.length); System.arraycopy(nc._ls, 0, _ls, sparseLen(), nc.sparseLen()); System.arraycopy(nc._xs, 0, _xs, sparseLen(), nc.sparseLen()); if (_id != null) { assert nc._id != null; _id = MemoryManager.arrayCopyOf(_id, _xs.length); System.arraycopy(nc._id, 0, _id, sparseLen(), nc.sparseLen()); for (int i = sparseLen(); i < sparseLen() + nc.sparseLen(); ++i) _id[i] += _len; } else assert nc._id == null; set_sparseLen(sparseLen() + nc.sparseLen()); set_len(_len + nc._len); nc._ls = null; nc._xs = null; nc._id = null; nc.set_sparseLen(nc.set_len(0)); assert sparseLen() <= _len; }