// Slow-path append data
private void append2slow() {
if (_len > Vec.CHUNK_SZ) throw new ArrayIndexOutOfBoundsException(_len);
assert _ds == null;
if (_len2 == _len) { // Check for sparse-ness now & then
int nzcnt = 0;
for (int i = 0; i < _len; i++) {
if (_ls[i] != 0) nzcnt++;
if (_xs[i] != 0) {
nzcnt = Vec.CHUNK_SZ;
break;
} // Only non-specials sparse
}
if (_len >= 32 && nzcnt * 8 <= _len) { // Heuristic for sparseness
_len = 0;
for (int i = 0; i < _len2; i++)
if (_ls[i] != 0) {
_xs[_len] = i; // Row number in xs
_ls[_len++] = _ls[i]; // Sparse value in ls
}
return; // Compressed, so lots of room now
}
}
_xs = _ls == null ? MemoryManager.malloc4(4) : MemoryManager.arrayCopyOf(_xs, _len << 1);
_ls = _ls == null ? MemoryManager.malloc8(4) : MemoryManager.arrayCopyOf(_ls, _len << 1);
}
// Slow-path append string
private void append2slowstr() {
// In case of all NAs and then a string, convert NAs to string NAs
if (_xs != null) {
_xs = null;
_ls = null;
alloc_str_indices(sparseLen());
Arrays.fill(_is, -1);
}
if (_is != null && _is.length > 0) {
// Check for sparseness
if (_id == null) {
int nzs = 0; // assume one non-null for the element currently being stored
for (int i : _is) if (i != -1) ++nzs;
if ((nzs + 1) * _sparseRatio < _len) set_sparse(nzs);
} else {
if ((_sparseRatio * (_sparseLen) >> 1) > _len) cancel_sparse();
else _id = MemoryManager.arrayCopyOf(_id, _sparseLen << 1);
}
_is = MemoryManager.arrayCopyOf(_is, sparseLen() << 1);
/* initialize the memory extension with -1s */
for (int i = sparseLen(); i < _is.length; i++) _is[i] = -1;
} else {
_is = MemoryManager.malloc4(4);
/* initialize everything with -1s */
for (int i = 0; i < _is.length; i++) _is[i] = -1;
if (sparse()) alloc_indices(4);
}
assert sparseLen() == 0 || _is.length > sparseLen()
: "_ls.length = " + _is.length + ", _len = " + sparseLen();
}
// Slow-path append data
private void append2slowd() {
assert _ls == null;
if (_ds != null && _ds.length > 0) {
if (_id == null) { // check for sparseness
int nzs = 0; // assume one non-zero for the element currently being stored
for (double d : _ds) if (d != 0) ++nzs;
if ((nzs + 1) * _sparseRatio < _len) set_sparse(nzs);
} else _id = MemoryManager.arrayCopyOf(_id, sparseLen() << 1);
_ds = MemoryManager.arrayCopyOf(_ds, sparseLen() << 1);
} else {
alloc_doubles(4);
if (sparse()) alloc_indices(4);
}
assert sparseLen() == 0 || _ds.length > sparseLen()
: "_ds.length = " + _ds.length + ", _len = " + sparseLen();
}
// Append all of 'nc' onto the current NewChunk. Kill nc.
public void add(NewChunk nc) {
assert _cidx >= 0;
assert sparseLen() <= _len;
assert nc.sparseLen() <= nc._len : "_len = " + nc.sparseLen() + ", _len2 = " + nc._len;
if (nc._len == 0) return;
if (_len == 0) {
_ls = nc._ls;
nc._ls = null;
_xs = nc._xs;
nc._xs = null;
_id = nc._id;
nc._id = null;
_ds = nc._ds;
nc._ds = null;
_is = nc._is;
nc._is = null;
_ss = nc._ss;
nc._ss = null;
set_sparseLen(nc.sparseLen());
set_len(nc._len);
return;
}
if (nc.sparse() != sparse()) { // for now, just make it dense
cancel_sparse();
nc.cancel_sparse();
}
if (_ds != null) throw H2O.fail();
while (sparseLen() + nc.sparseLen() >= _xs.length)
_xs = MemoryManager.arrayCopyOf(_xs, _xs.length << 1);
_ls = MemoryManager.arrayCopyOf(_ls, _xs.length);
System.arraycopy(nc._ls, 0, _ls, sparseLen(), nc.sparseLen());
System.arraycopy(nc._xs, 0, _xs, sparseLen(), nc.sparseLen());
if (_id != null) {
assert nc._id != null;
_id = MemoryManager.arrayCopyOf(_id, _xs.length);
System.arraycopy(nc._id, 0, _id, sparseLen(), nc.sparseLen());
for (int i = sparseLen(); i < sparseLen() + nc.sparseLen(); ++i) _id[i] += _len;
} else assert nc._id == null;
set_sparseLen(sparseLen() + nc.sparseLen());
set_len(_len + nc._len);
nc._ls = null;
nc._xs = null;
nc._id = null;
nc.set_sparseLen(nc.set_len(0));
assert sparseLen() <= _len;
}
// Slow-path append data
private void append2slowUUID() {
if (_ds == null && _ls != null) { // This can happen for columns with all NAs and then a UUID
_xs = null;
alloc_doubles(sparseLen());
Arrays.fill(_ls, C16Chunk._LO_NA);
Arrays.fill(_ds, Double.longBitsToDouble(C16Chunk._HI_NA));
}
if (_ls != null && _ls.length > 0) {
_ls = MemoryManager.arrayCopyOf(_ls, sparseLen() << 1);
_ds = MemoryManager.arrayCopyOf(_ds, sparseLen() << 1);
} else {
alloc_mantissa(4);
alloc_doubles(4);
}
assert sparseLen() == 0 || _ls.length > sparseLen()
: "_ls.length = " + _ls.length + ", _len = " + sparseLen();
}
private void append_ss(String str) {
if (_ss == null) {
_ss = MemoryManager.malloc1((str.length() + 1) * 4);
}
while (_ss.length < (_sslen + str.length() + 1)) {
_ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);
}
for (byte b : str.getBytes()) _ss[_sslen++] = b;
_ss[_sslen++] = (byte) 0; // for trailing 0;
}
// Slow-path append data
private void append2slow() {
if (sparseLen() > FileVec.DFLT_CHUNK_SIZE)
throw new ArrayIndexOutOfBoundsException(sparseLen());
assert _ds == null;
if (_ls != null && _ls.length > 0) {
if (_id == null) { // check for sparseness
int nzs = 0;
for (int i = 0; i < _ls.length; ++i) if (_ls[i] != 0 || _xs[i] != 0) ++nzs;
if ((nzs + 1) * _sparseRatio < _len) {
set_sparse(nzs);
assert sparseLen() == 0 || sparseLen() <= _ls.length
: "_len = "
+ sparseLen()
+ ", _ls.length = "
+ _ls.length
+ ", nzs = "
+ nzs
+ ", len2 = "
+ _len;
assert _id.length == _ls.length;
assert sparseLen() <= _len;
return;
}
} else {
// verify we're still sufficiently sparse
if ((_sparseRatio * (sparseLen()) >> 1) > _len) cancel_sparse();
else _id = MemoryManager.arrayCopyOf(_id, sparseLen() << 1);
}
_ls = MemoryManager.arrayCopyOf(_ls, sparseLen() << 1);
_xs = MemoryManager.arrayCopyOf(_xs, sparseLen() << 1);
} else {
alloc_mantissa(4);
alloc_exponent(4);
if (_id != null) alloc_indices(4);
}
assert sparseLen() == 0 || sparseLen() < _ls.length
: "_len = " + sparseLen() + ", _ls.length = " + _ls.length;
assert _id == null || _id.length == _ls.length;
assert sparseLen() <= _len;
}
private void append_ss(String str) {
byte[] bytes = str.getBytes(Charsets.UTF_8);
// Allocate memory if necessary
if (_ss == null) _ss = MemoryManager.malloc1((bytes.length + 1) * 4);
while (_ss.length < (_sslen + bytes.length + 1))
_ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);
// Copy bytes to _ss
for (byte b : bytes) _ss[_sslen++] = b;
_ss[_sslen++] = (byte) 0; // for trailing 0;
}
private void append_ss(BufferedString str) {
int strlen = str.length();
int off = str.getOffset();
byte b[] = str.getBuffer();
if (_ss == null) {
_ss = MemoryManager.malloc1((strlen + 1) * 4);
}
while (_ss.length < (_sslen + strlen + 1)) {
_ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);
}
for (int i = off; i < off + strlen; i++) _ss[_sslen++] = b[i];
_ss[_sslen++] = (byte) 0; // for trailing 0;
}
public double[] denormalizeBeta(double[] beta) {
if (beta.length != fullN() + 1) System.out.println("haha");
assert beta.length == fullN() + 1 : "beta len = " + beta.length;
beta = MemoryManager.arrayCopyOf(beta, beta.length);
if (_predictor_transform == DataInfo.TransformType.STANDARDIZE) {
double norm = 0.0; // Reverse any normalization on the intercept
// denormalize only the numeric coefs (categoricals are not normalized)
final int numoff = numStart();
for (int i = numoff; i < beta.length - 1; i++) {
double b = beta[i] * _normMul[i - numoff];
norm += b * _normSub[i - numoff]; // Also accumulate the intercept adjustment
beta[i] = b;
}
beta[beta.length - 1] -= norm;
}
return beta;
}
public double[] denormalizeBeta(double[] beta) {
int N = fullN() + 1;
assert (beta.length % N) == 0 : "beta len = " + beta.length + " expected multiple of" + N;
int nclasses = beta.length / N;
beta = MemoryManager.arrayCopyOf(beta, beta.length);
if (_predictor_transform == DataInfo.TransformType.STANDARDIZE) {
for (int c = 0; c < nclasses; ++c) {
int off = N * c;
double norm = 0.0; // Reverse any normalization on the intercept
// denormalize only the numeric coefs (categoricals are not normalized)
final int numoff = numStart();
for (int i = numoff; i < N - 1; i++) {
if (isInteractionVec(i)) continue; // interactions are not normalized!
double b = beta[off + i] * _normMul[i - numoff];
norm += b * _normSub[i - numoff]; // Also accumulate the intercept adjustment
beta[off + i] = b;
}
beta[off + N - 1] -= norm;
}
}
return beta;
}
// Slow-path append data
private void append2slowd() {
if (_len > Vec.CHUNK_SZ) throw new ArrayIndexOutOfBoundsException(_len);
assert _ls == null && _len2 == _len;
_ds = _ds == null ? MemoryManager.malloc8d(4) : MemoryManager.arrayCopyOf(_ds, _len << 1);
}