/**
* Extract (sparse) rows from given chunks. Note: 0 remains 0 - _normSub of DataInfo isn't used
* (mean shift during standarization is not reverted) - UNLESS offset is specified (for GLM only)
* Essentially turns the dataset 90 degrees.
*
* @param chunks - chunk of dataset
* @param offset - adjustment for 0s if running with on-the-fly standardization (i.e. zeros are
* not really zeros because of centering)
* @return array of sparse rows
*/
public final Row[] extractSparseRows(Chunk[] chunks, double offset) {
Row[] rows = new Row[chunks[0]._len];
for (int i = 0; i < rows.length; ++i) {
rows[i] = new Row(true, Math.min(_nums, 16), _cats, _responses, offset);
rows[i].rid = chunks[0].start() + i;
if (_offset) {
rows[i].offset = chunks[offsetChunkId()].atd(i);
if (Double.isNaN(rows[i].offset)) rows[i].bad = true;
}
if (_weights) {
rows[i].weight = chunks[weightChunkId()].atd(i);
if (Double.isNaN(rows[i].weight)) rows[i].bad = true;
}
}
// categoricals
for (int i = 0; i < _cats; ++i) {
for (int r = 0; r < chunks[0]._len; ++r) {
Row row = rows[r];
if (row.bad) continue;
if (chunks[i].isNA(r)) {
if (_skipMissing) {
row.bad = true;
} else
row.binIds[row.nBins++] =
_catOffsets[i + 1] - 1; // missing value turns into extra (last) factor
} else {
int c = getCategoricalId(i, (int) chunks[i].at8(r));
if (c >= 0) row.binIds[row.nBins++] = c;
}
}
}
int numStart = numStart();
// generic numbers
for (int cid = 0; cid < _nums; ++cid) {
Chunk c = chunks[_cats + cid];
int oldRow = -1;
for (int r = c.nextNZ(-1); r < c._len; r = c.nextNZ(r)) {
if (c.atd(r) == 0) continue;
assert r > oldRow;
oldRow = r;
Row row = rows[r];
if (row.bad) continue;
if (c.isNA(r)) row.bad = _skipMissing;
double d = c.atd(r);
if (_normMul != null) d *= _normMul[cid];
row.addNum(cid + numStart, d);
}
}
// response(s)
for (int i = 1; i <= _responses; ++i) {
Chunk rChunk = chunks[responseChunkId()];
for (int r = 0; r < chunks[0]._len; ++r) {
Row row = rows[r];
if (row.bad) continue;
row.response[row.response.length - i] = rChunk.atd(r);
if (_normRespMul != null) {
row.response[i - 1] = (row.response[i - 1] - _normRespSub[i - 1]) * _normRespMul[i - 1];
}
if (Double.isNaN(row.response[row.response.length - i])) row.bad = true;
}
}
return rows;
}
/**
* Extract (sparse) rows from given chunks. Note: 0 remains 0 - _normSub of DataInfo isn't used
* (mean shift during standarization is not reverted) - UNLESS offset is specified (for GLM only)
* Essentially turns the dataset 90 degrees.
*
* @param chunks - chunk of dataset
* @return array of sparse rows
*/
public final Row[] extractSparseRows(Chunk[] chunks) {
Row[] rows = new Row[chunks[0]._len];
long startOff = chunks[0].start();
for (int i = 0; i < rows.length; ++i) {
rows[i] =
new Row(
true,
Math.min(_nums, 16),
_cats,
_responses,
i,
startOff); // if sparse, _nums is the correct number of nonzero values! i.e., do not
// use numNums()
rows[i].rid = chunks[0].start() + i;
if (_offset) {
rows[i].offset = chunks[offsetChunkId()].atd(i);
if (Double.isNaN(rows[i].offset)) rows[i].bad = true;
}
if (_weights) {
rows[i].weight = chunks[weightChunkId()].atd(i);
if (Double.isNaN(rows[i].weight)) rows[i].bad = true;
}
if (_skipMissing) {
int N = _cats + _nums;
for (int c = 0; c < N; ++c) if (chunks[c].isNA(i)) rows[i].bad = true;
}
}
// categoricals
for (int i = 0; i < _cats; ++i) {
for (int r = 0; r < chunks[0]._len; ++r) {
Row row = rows[r];
if (row.bad) continue;
int cid = getCategoricalId(i, chunks[i].isNA(r) ? _catModes[i] : (int) chunks[i].at8(r));
if (cid >= 0) row.binIds[row.nBins++] = cid;
}
}
// generic numbers + interactions
int interactionOffset = 0;
for (int cid = 0; cid < _nums; ++cid) {
Chunk c = chunks[_cats + cid];
int oldRow = -1;
if (c
instanceof
InteractionWrappedVec
.InteractionWrappedChunk) { // for each row, only 1 value in an interaction is 'hot'
// all other values are off (i.e., are 0)
for (int r = 0;
r < c._len;
++r) { // the vec is "vertically" dense and "horizontally" sparse (i.e., every row has
// one, and only one, value)
Row row = rows[r];
if (row.bad) continue;
if (c.isNA(r)) row.bad = _skipMissing;
int cidVirtualOffset =
getInteractionOffset(
chunks, _cats + cid, r); // the "virtual" offset into the hot-expanded interaction
row.addNum(
_numOffsets[cid] + cidVirtualOffset,
c.atd(r)); // FIXME: if this produces a "true" NA then should sub with mean? with?
}
interactionOffset += nextNumericIdx(cid);
} else {
for (int r = c.nextNZ(-1); r < c._len; r = c.nextNZ(r)) {
if (c.atd(r) == 0) continue;
assert r > oldRow;
oldRow = r;
Row row = rows[r];
if (row.bad) continue;
if (c.isNA(r)) row.bad = _skipMissing;
double d = c.atd(r);
if (Double.isNaN(d)) d = _numMeans[cid];
if (_normMul != null) d *= _normMul[interactionOffset];
row.addNum(_numOffsets[cid], d);
}
interactionOffset++;
}
}
// response(s)
for (int i = 1; i <= _responses; ++i) {
int rid = responseChunkId(i - 1);
Chunk rChunk = chunks[rid];
for (int r = 0; r < chunks[0]._len; ++r) {
Row row = rows[r];
if (row.bad) continue;
row.response[i - 1] = rChunk.atd(r);
if (_normRespMul != null) {
row.response[i - 1] = (row.response[i - 1] - _normRespSub[i - 1]) * _normRespMul[i - 1];
}
if (Double.isNaN(row.response[row.response.length - i])) row.bad = true;
}
}
return rows;
}