Пример #1
0
  // Slow-path append string
  private void append2slowstr() {
    // In case of all NAs and then a string, convert NAs to string NAs
    if (_xs != null) {
      _xs = null;
      _ls = null;
      alloc_str_indices(sparseLen());
      Arrays.fill(_is, -1);
    }

    if (_is != null && _is.length > 0) {
      // Check for sparseness
      if (_id == null) {
        int nzs = 0; // assume one non-null for the element currently being stored
        for (int i : _is) if (i != -1) ++nzs;
        if ((nzs + 1) * _sparseRatio < _len) set_sparse(nzs);
      } else {
        if ((_sparseRatio * (_sparseLen) >> 1) > _len) cancel_sparse();
        else _id = MemoryManager.arrayCopyOf(_id, _sparseLen << 1);
      }

      _is = MemoryManager.arrayCopyOf(_is, sparseLen() << 1);
      /* initialize the memory extension with -1s */
      for (int i = sparseLen(); i < _is.length; i++) _is[i] = -1;
    } else {
      _is = MemoryManager.malloc4(4);
      /* initialize everything with -1s */
      for (int i = 0; i < _is.length; i++) _is[i] = -1;
      if (sparse()) alloc_indices(4);
    }
    assert sparseLen() == 0 || _is.length > sparseLen()
        : "_ls.length = " + _is.length + ", _len = " + sparseLen();
  }
Пример #2
0
 // Slow-path append data
 private void append2slow() {
   if (_len > Vec.CHUNK_SZ) throw new ArrayIndexOutOfBoundsException(_len);
   assert _ds == null;
   if (_len2 == _len) { // Check for sparse-ness now & then
     int nzcnt = 0;
     for (int i = 0; i < _len; i++) {
       if (_ls[i] != 0) nzcnt++;
       if (_xs[i] != 0) {
         nzcnt = Vec.CHUNK_SZ;
         break;
       } // Only non-specials sparse
     }
     if (_len >= 32 && nzcnt * 8 <= _len) { // Heuristic for sparseness
       _len = 0;
       for (int i = 0; i < _len2; i++)
         if (_ls[i] != 0) {
           _xs[_len] = i; // Row number in xs
           _ls[_len++] = _ls[i]; // Sparse value in ls
         }
       return; // Compressed, so lots of room now
     }
   }
   _xs = _ls == null ? MemoryManager.malloc4(4) : MemoryManager.arrayCopyOf(_xs, _len << 1);
   _ls = _ls == null ? MemoryManager.malloc8(4) : MemoryManager.arrayCopyOf(_ls, _len << 1);
 }
Пример #3
0
 public Submodel(
     double lambda,
     double[] beta,
     double[] norm_beta,
     long run_time,
     int iteration,
     boolean sparseCoef) {
   this.lambda_value = lambda;
   this.run_time = run_time;
   this.iteration = iteration;
   int r = 0;
   if (beta != null) {
     final double[] b = norm_beta != null ? norm_beta : beta;
     // grab the indeces of non-zero coefficients
     for (double d : beta) if (d != 0) ++r;
     idxs = MemoryManager.malloc4(sparseCoef ? r : beta.length);
     int j = 0;
     for (int i = 0; i < beta.length; ++i) if (!sparseCoef || beta[i] != 0) idxs[j++] = i;
     j = 0;
     this.beta = MemoryManager.malloc8d(idxs.length);
     for (int i : idxs) this.beta[j++] = beta[i];
     if (norm_beta != null) {
       j = 0;
       this.norm_beta = MemoryManager.malloc8d(idxs.length);
       for (int i : idxs) this.norm_beta[j++] = norm_beta[i];
     }
   } else idxs = null;
   rank = r;
   this.sparseCoef = sparseCoef;
 }
Пример #4
0
 protected void cancel_sparse() {
   if (sparseLen() != _len) {
     if (_is != null) {
       int[] is = MemoryManager.malloc4(_len);
       for (int i = 0; i < _len; i++) is[i] = -1;
       for (int i = 0; i < sparseLen(); i++) is[_id[i]] = _is[i];
       _is = is;
     } else if (_ds == null) {
       int[] xs = MemoryManager.malloc4(_len);
       long[] ls = MemoryManager.malloc8(_len);
       for (int i = 0; i < sparseLen(); ++i) {
         xs[_id[i]] = _xs[i];
         ls[_id[i]] = _ls[i];
       }
       _xs = xs;
       _ls = ls;
     } else {
       double[] ds = MemoryManager.malloc8d(_len);
       for (int i = 0; i < sparseLen(); ++i) ds[_id[i]] = _ds[i];
       _ds = ds;
     }
     set_sparseLen(_len);
   }
   _id = null;
 }
Пример #5
0
 public Row(boolean sparse, int nNums, int nBins, int nresponses, double etaOffset) {
   binIds = MemoryManager.malloc4(nBins);
   numVals = MemoryManager.malloc8d(nNums);
   response = MemoryManager.malloc8d(nresponses);
   if (sparse) numIds = MemoryManager.malloc4(nNums);
   this.etaOffset = etaOffset;
   this.nNums = sparse ? 0 : nNums;
 }
Пример #6
0
 private void cancel_sparse() {
   long ls[] = MemoryManager.malloc8(_len2 + 1);
   for (int i = 0; i < _len; i++) // Inflate ls to hold values
   ls[_xs[i]] = _ls[i];
   _ls = ls;
   _xs = MemoryManager.malloc4(_len2 + 1);
   _len = _len2; // Not compressed now!
 }
Пример #7
0
 public Row(boolean sparse, int nNums, int nBins, int nresponses, int i, long start) {
   binIds = MemoryManager.malloc4(nBins);
   numVals = MemoryManager.malloc8d(nNums);
   response = MemoryManager.malloc8d(nresponses);
   if (sparse) numIds = MemoryManager.malloc4(nNums);
   this.nNums = sparse ? 0 : nNums;
   cid = i;
   rid = start + i;
 }
Пример #8
0
 private void append_ss(String str) {
   if (_ss == null) {
     _ss = MemoryManager.malloc1((str.length() + 1) * 4);
   }
   while (_ss.length < (_sslen + str.length() + 1)) {
     _ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);
   }
   for (byte b : str.getBytes()) _ss[_sslen++] = b;
   _ss[_sslen++] = (byte) 0; // for trailing 0;
 }
Пример #9
0
 public void setPredictorTransform(TransformType t) {
   _predictor_transform = t;
   if (t == TransformType.NONE) {
     _normMul = null;
     _normSub = null;
   } else {
     _normMul = MemoryManager.malloc8d(_nums);
     _normSub = MemoryManager.malloc8d(_nums);
     setTransform(t, _normMul, _normSub, _cats, _nums);
   }
 }
Пример #10
0
 public void setResponseTransform(TransformType t) {
   _response_transform = t;
   if (t == TransformType.NONE) {
     _normRespMul = null;
     _normRespSub = null;
   } else {
     _normRespMul = MemoryManager.malloc8d(_responses);
     _normRespSub = MemoryManager.malloc8d(_responses);
     setTransform(t, _normRespMul, _normRespSub, _adaptedFrame.numCols() - _responses, _responses);
   }
 }
Пример #11
0
  private void append_ss(String str) {
    byte[] bytes = str.getBytes(Charsets.UTF_8);

    // Allocate memory if necessary
    if (_ss == null) _ss = MemoryManager.malloc1((bytes.length + 1) * 4);
    while (_ss.length < (_sslen + bytes.length + 1))
      _ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);

    // Copy bytes to _ss
    for (byte b : bytes) _ss[_sslen++] = b;
    _ss[_sslen++] = (byte) 0; // for trailing 0;
  }
Пример #12
0
 @Override
 NewChunk inflate_impl(NewChunk nc) {
   double dx = Math.log10(_scale);
   assert DParseTask.fitsIntoInt(dx);
   Arrays.fill(nc._xs = MemoryManager.malloc4(_len), (int) dx);
   nc._ls = MemoryManager.malloc8(_len);
   for (int i = 0; i < _len; i++) {
     int res = UDP.get2(_mem, (i << 1) + OFF);
     if (res == C2Chunk._NA) nc.setNA_impl2(i);
     else nc._ls[i] = res + _bias;
   }
   return nc;
 }
Пример #13
0
  private void append_ss(BufferedString str) {
    int strlen = str.length();
    int off = str.getOffset();
    byte b[] = str.getBuffer();

    if (_ss == null) {
      _ss = MemoryManager.malloc1((strlen + 1) * 4);
    }
    while (_ss.length < (_sslen + strlen + 1)) {
      _ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);
    }
    for (int i = off; i < off + strlen; i++) _ss[_sslen++] = b[i];
    _ss[_sslen++] = (byte) 0; // for trailing 0;
  }
Пример #14
0
 private final double[] contractVec(double[] beta, final int[] activeCols) {
   double[] res = MemoryManager.malloc8d(activeCols.length + 1);
   int i = 0;
   for (int c : activeCols) res[i++] = beta[c];
   res[res.length - 1] = beta[beta.length - 1];
   return res;
 }
Пример #15
0
 // Compute a sparse float buffer
 private byte[] bufD(final int valsz) {
   int log = 0;
   while ((1 << log) < valsz) ++log;
   assert (1 << log) == valsz;
   final int ridsz = _len >= 65535 ? 4 : 2;
   final int elmsz = ridsz + valsz;
   int off = CXDChunk._OFF;
   byte[] buf = MemoryManager.malloc1(off + sparseLen() * elmsz, true);
   for (int i = 0; i < sparseLen(); i++, off += elmsz) {
     if (ridsz == 2) UnsafeUtils.set2(buf, off, (short) _id[i]);
     else UnsafeUtils.set4(buf, off, _id[i]);
     final double dval =
         _ds == null ? isNA2(i) ? Double.NaN : _ls[i] * PrettyPrint.pow10(_xs[i]) : _ds[i];
     switch (valsz) {
       case 4:
         UnsafeUtils.set4f(buf, off + ridsz, (float) dval);
         break;
       case 8:
         UnsafeUtils.set8d(buf, off + ridsz, dval);
         break;
       default:
         throw H2O.fail();
     }
   }
   assert off == buf.length;
   return buf;
 }
Пример #16
0
 public ModelDataAdaptor(OldModel M, int yCol, int[] cols, int[][] catMap) {
   this.M = M;
   _yCol = yCol;
   _row = MemoryManager.malloc8d(cols.length);
   _xCols = cols;
   _catMap = catMap;
 }
Пример #17
0
 private double[] setNewBeta(final double[] newBeta) {
   final double[] fullBeta;
   if (_activeCols != null) {
     fullBeta = MemoryManager.malloc8d(_dinfo.fullN() + 1);
     int j = 0;
     for (int i : _activeCols) fullBeta[i] = newBeta[j++];
     assert j == newBeta.length - 1;
     fullBeta[fullBeta.length - 1] = newBeta[j];
   } else {
     assert newBeta.length == _dinfo.fullN() + 1;
     fullBeta = newBeta;
   }
   final double[] newBetaDeNorm;
   if (_dinfo._standardize) {
     newBetaDeNorm = fullBeta.clone();
     double norm = 0.0; // Reverse any normalization on the intercept
     // denormalize only the numeric coefs (categoricals are not normalized)
     final int numoff = _dinfo.numStart();
     for (int i = numoff; i < fullBeta.length - 1; i++) {
       double b = newBetaDeNorm[i] * _dinfo._normMul[i - numoff];
       norm += b * _dinfo._normSub[i - numoff]; // Also accumulate the intercept adjustment
       newBetaDeNorm[i] = b;
     }
     newBetaDeNorm[newBetaDeNorm.length - 1] -= norm;
   } else newBetaDeNorm = null;
   _model.setLambdaSubmodel(
       _lambdaIdx,
       newBetaDeNorm == null ? fullBeta : newBetaDeNorm,
       newBetaDeNorm == null ? null : fullBeta,
       (_iter + 1));
   _model.clone().update(self());
   return fullBeta;
 }
Пример #18
0
 // Slow-path append data
 private void append2slowd() {
   assert _ls == null;
   if (_ds != null && _ds.length > 0) {
     if (_id == null) { // check for sparseness
       int nzs = 0; // assume one non-zero for the element currently being stored
       for (double d : _ds) if (d != 0) ++nzs;
       if ((nzs + 1) * _sparseRatio < _len) set_sparse(nzs);
     } else _id = MemoryManager.arrayCopyOf(_id, sparseLen() << 1);
     _ds = MemoryManager.arrayCopyOf(_ds, sparseLen() << 1);
   } else {
     alloc_doubles(4);
     if (sparse()) alloc_indices(4);
   }
   assert sparseLen() == 0 || _ds.length > sparseLen()
       : "_ds.length = " + _ds.length + ", _len = " + sparseLen();
 }
Пример #19
0
 // filter the current active columns using the strong rules
 // note: strong rules are update so tha they keep all previous coefficients in, to prevent issues
 // with line-search
 private int[] activeCols(final double l1, final double l2, final double[] grad) {
   final double rhs = alpha[0] * (2 * l1 - l2);
   int[] cols = MemoryManager.malloc4(_dinfo.fullN());
   int selected = 0;
   int j = 0;
   if (_activeCols == null) _activeCols = new int[] {-1};
   for (int i = 0; i < _dinfo.fullN(); ++i)
     if ((j < _activeCols.length && i == _activeCols[j]) || grad[i] > rhs || grad[i] < -rhs) {
       cols[selected++] = i;
       if (j < _activeCols.length && i == _activeCols[j]) ++j;
     }
   if (!strong_rules_enabled || selected == _dinfo.fullN()) {
     _activeCols = null;
     _activeData._adaptedFrame = _dinfo._adaptedFrame;
     _activeData = _dinfo;
   } else {
     _activeCols = Arrays.copyOf(cols, selected);
     _activeData = _dinfo.filterExpandedColumns(_activeCols);
   }
   Log.info(
       "GLM2 strong rule at lambda="
           + l1
           + ", got "
           + selected
           + " active cols out of "
           + _dinfo.fullN()
           + " total.");
   return _activeCols;
 }
Пример #20
0
 // Compute a compressed double buffer
 private Chunk chunkD() {
   HashMap<Long, Byte> hs = new HashMap<>(CUDChunk.MAX_UNIQUES);
   Byte dummy = 0;
   final byte[] bs = MemoryManager.malloc1(_len * 8, true);
   int j = 0;
   boolean fitsInUnique = true;
   for (int i = 0; i < _len; ++i) {
     double d = 0;
     if (_id == null || _id.length == 0 || (j < _id.length && _id[j] == i)) {
       d =
           _ds != null
               ? _ds[j]
               : (isNA2(j) || isCategorical(j)) ? Double.NaN : _ls[j] * PrettyPrint.pow10(_xs[j]);
       ++j;
     }
     if (fitsInUnique) {
       if (hs.size() < CUDChunk.MAX_UNIQUES) // still got space
       hs.put(
             Double.doubleToLongBits(d),
             dummy); // store doubles as longs to avoid NaN comparison issues during extraction
       else
         fitsInUnique =
             (hs.size() == CUDChunk.MAX_UNIQUES)
                 && // full, but might not need more space because of repeats
                 hs.containsKey(Double.doubleToLongBits(d));
     }
     UnsafeUtils.set8d(bs, 8 * i, d);
   }
   assert j == sparseLen() : "j = " + j + ", _len = " + sparseLen();
   if (fitsInUnique && CUDChunk.computeByteSize(hs.size(), len()) < 0.8 * bs.length)
     return new CUDChunk(bs, hs, len());
   else return new C8DChunk(bs);
 }
Пример #21
0
 private double[] scoreRow(Row r, double o, double[] preds) {
   if (_parms._family == Family.multinomial) {
     if (_eta == null) _eta = new ThreadLocal<>();
     double[] eta = _eta.get();
     if (eta == null) _eta.set(eta = MemoryManager.malloc8d(_output.nclasses()));
     final double[][] bm = _output._global_beta_multinomial;
     double sumExp = 0;
     double maxRow = 0;
     for (int c = 0; c < bm.length; ++c) {
       eta[c] = r.innerProduct(bm[c]) + o;
       if (eta[c] > maxRow) maxRow = eta[c];
     }
     for (int c = 0; c < bm.length; ++c) sumExp += eta[c] = Math.exp(eta[c] - maxRow); // intercept
     sumExp = 1.0 / sumExp;
     for (int c = 0; c < bm.length; ++c) preds[c + 1] = eta[c] * sumExp;
     preds[0] = ArrayUtils.maxIndex(eta);
   } else {
     double mu = _parms.linkInv(r.innerProduct(beta()) + o);
     if (_parms._family == Family.binomial) { // threshold for prediction
       preds[0] = mu >= defaultThreshold() ? 1 : 0;
       preds[1] = 1.0 - mu; // class 0
       preds[2] = mu; // class 1
     } else preds[0] = mu;
   }
   return preds;
 }
Пример #22
0
  // Append all of 'nc' onto the current NewChunk.  Kill nc.
  public void add(NewChunk nc) {
    assert _cidx >= 0;
    assert sparseLen() <= _len;
    assert nc.sparseLen() <= nc._len : "_len = " + nc.sparseLen() + ", _len2 = " + nc._len;
    if (nc._len == 0) return;
    if (_len == 0) {
      _ls = nc._ls;
      nc._ls = null;
      _xs = nc._xs;
      nc._xs = null;
      _id = nc._id;
      nc._id = null;
      _ds = nc._ds;
      nc._ds = null;
      _is = nc._is;
      nc._is = null;
      _ss = nc._ss;
      nc._ss = null;
      set_sparseLen(nc.sparseLen());
      set_len(nc._len);
      return;
    }
    if (nc.sparse() != sparse()) { // for now, just make it dense
      cancel_sparse();
      nc.cancel_sparse();
    }
    if (_ds != null) throw H2O.fail();
    while (sparseLen() + nc.sparseLen() >= _xs.length)
      _xs = MemoryManager.arrayCopyOf(_xs, _xs.length << 1);
    _ls = MemoryManager.arrayCopyOf(_ls, _xs.length);
    System.arraycopy(nc._ls, 0, _ls, sparseLen(), nc.sparseLen());
    System.arraycopy(nc._xs, 0, _xs, sparseLen(), nc.sparseLen());
    if (_id != null) {
      assert nc._id != null;
      _id = MemoryManager.arrayCopyOf(_id, _xs.length);
      System.arraycopy(nc._id, 0, _id, sparseLen(), nc.sparseLen());
      for (int i = sparseLen(); i < sparseLen() + nc.sparseLen(); ++i) _id[i] += _len;
    } else assert nc._id == null;

    set_sparseLen(sparseLen() + nc.sparseLen());
    set_len(_len + nc._len);
    nc._ls = null;
    nc._xs = null;
    nc._id = null;
    nc.set_sparseLen(nc.set_len(0));
    assert sparseLen() <= _len;
  }
Пример #23
0
 private final double[] expandVec(double[] beta, final int[] activeCols) {
   if (activeCols == null) return beta;
   double[] res = MemoryManager.malloc8d(_dinfo.fullN() + 1);
   int i = 0;
   for (int c : activeCols) res[c] = beta[i++];
   res[res.length - 1] = beta[beta.length - 1];
   return res;
 }
Пример #24
0
 void addNullSubmodel(double lmax, double icept, GLMValidation val) {
   assert _submodels == null;
   double[] beta = MemoryManager.malloc8d(_coefficient_names.length);
   beta[beta.length - 1] = icept;
   _submodels =
       new Submodel[] {new Submodel(lmax, beta, beta, 0, 0, _coefficient_names.length > 750)};
   _submodels[0].validation = val;
 }
Пример #25
0
 Col(String s, int rows, boolean isClass) {
   _name = s;
   _isClass = isClass;
   _rawB = MemoryManager.malloc1(rows);
   _isFloat = false;
   _isByte = true;
   _colBinLimit = 0;
 }
Пример #26
0
 // Slow-path append data
 private void append2slowUUID() {
   if (_ds == null && _ls != null) { // This can happen for columns with all NAs and then a UUID
     _xs = null;
     alloc_doubles(sparseLen());
     Arrays.fill(_ls, C16Chunk._LO_NA);
     Arrays.fill(_ds, Double.longBitsToDouble(C16Chunk._HI_NA));
   }
   if (_ls != null && _ls.length > 0) {
     _ls = MemoryManager.arrayCopyOf(_ls, sparseLen() << 1);
     _ds = MemoryManager.arrayCopyOf(_ds, sparseLen() << 1);
   } else {
     alloc_mantissa(4);
     alloc_doubles(4);
   }
   assert sparseLen() == 0 || _ls.length > sparseLen()
       : "_ls.length = " + _ls.length + ", _len = " + sparseLen();
 }
Пример #27
0
 @Override
 public byte[] atomic(byte[] bits1) {
   byte[] mem = DKV.get(_key).get();
   int len = Math.max(_dst_off + mem.length, bits1 == null ? 0 : bits1.length);
   byte[] bits2 = MemoryManager.malloc1(len);
   if (bits1 != null) System.arraycopy(bits1, 0, bits2, 0, bits1.length);
   System.arraycopy(mem, 0, bits2, _dst_off, mem.length);
   return bits2;
 }
Пример #28
0
 Col(String s, int rows, boolean isClass, int binLimit, boolean isFloat) {
   _name = s;
   _isFloat = isFloat;
   _isClass = isClass;
   _colBinLimit = binLimit;
   _isByte = false;
   _raw = MemoryManager.malloc4f(rows);
   _ignored = false;
 }
Пример #29
0
 private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
   try {
     if (fs == null) return;
     for (FileStatus file : fs.listStatus(p)) {
       Path pfs = file.getPath();
       if (file.isDir()) {
         addFolder(fs, pfs, succeeded, failed);
       } else {
         Key k = Key.make(pfs.toString());
         long size = file.getLen();
         Value val = null;
         if (pfs.getName().endsWith(Extensions.JSON)) {
           JsonParser parser = new JsonParser();
           JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
           JsonElement v = json.get(Constants.VERSION);
           if (v == null) throw new RuntimeException("Missing version");
           JsonElement type = json.get(Constants.TYPE);
           if (type == null) throw new RuntimeException("Missing type");
           Class c = Class.forName(type.getAsString());
           OldModel model = (OldModel) c.newInstance();
           model.fromJson(json);
         } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
           FSDataInputStream s = fs.open(pfs);
           int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
           byte[] mem = MemoryManager.malloc1(sz);
           s.readFully(mem);
           // Convert to a ValueArray (hope it fits in 1Meg!)
           ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
           val = new Value(k, ary, Value.HDFS);
         } else if (size >= 2 * ValueArray.CHUNK_SZ) {
           val =
               new Value(
                   k,
                   new ValueArray(k, size),
                   Value.HDFS); // ValueArray byte wrapper over a large file
         } else {
           val = new Value(k, (int) size, Value.HDFS); // Plain Value
           val.setdsk();
         }
         DKV.put(k, val);
         Log.info("PersistHdfs: DKV.put(" + k + ")");
         JsonObject o = new JsonObject();
         o.addProperty(Constants.KEY, k.toString());
         o.addProperty(Constants.FILE, pfs.toString());
         o.addProperty(Constants.VALUE_SIZE, file.getLen());
         succeeded.add(o);
       }
     }
   } catch (Exception e) {
     Log.err(e);
     JsonObject o = new JsonObject();
     o.addProperty(Constants.FILE, p.toString());
     o.addProperty(Constants.ERROR, e.getMessage());
     failed.add(o);
   }
 }
Пример #30
0
 // private constructor called by filterExpandedColumns
 private DataInfo(
     DataInfo dinfo,
     Frame fr,
     double[] normMul,
     double[] normSub,
     int[][] catLevels,
     int[] catModes) {
   _fullCatOffsets = dinfo._catOffsets;
   if (!dinfo._useAllFactorLevels) {
     _fullCatOffsets = dinfo._catOffsets.clone();
     for (int i = 0; i < _fullCatOffsets.length; ++i)
       _fullCatOffsets[i] += i; // add for the skipped zeros.
   }
   _offset = dinfo._offset;
   _weights = dinfo._weights;
   _fold = dinfo._fold;
   _valid = false;
   _interactions = dinfo._interactions;
   _interactionVecs = dinfo._interactionVecs;
   assert dinfo._predictor_transform != null;
   assert dinfo._response_transform != null;
   _predictor_transform = dinfo._predictor_transform;
   _response_transform = dinfo._response_transform;
   _skipMissing = dinfo._skipMissing;
   _imputeMissing = dinfo._imputeMissing;
   _adaptedFrame = fr;
   _catOffsets = MemoryManager.malloc4(catLevels.length + 1);
   _catMissing = new boolean[catLevels.length];
   Arrays.fill(_catMissing, !(dinfo._imputeMissing || dinfo._skipMissing));
   int s = 0;
   for (int i = 0; i < catLevels.length; ++i) {
     _catOffsets[i] = s;
     s += catLevels[i].length;
   }
   _catLvls = catLevels;
   _catOffsets[_catOffsets.length - 1] = s;
   _responses = dinfo._responses;
   _cats = catLevels.length;
   _nums =
       fr.numCols()
           - _cats
           - dinfo._responses
           - (_offset ? 1 : 0)
           - (_weights ? 1 : 0)
           - (_fold ? 1 : 0);
   _numOffsets = _nums == 0 ? new int[0] : dinfo._numOffsets.clone();
   int diff = _numOffsets.length > 0 ? _numOffsets[0] - s : 0;
   for (int i = 0; i < _numOffsets.length; i++) // need to shift everyone down by the offset!
   _numOffsets[i] -= diff;
   _useAllFactorLevels = true; // dinfo._useAllFactorLevels;
   _numMeans = new double[_nums];
   _normMul = normMul;
   _normSub = normSub;
   _catModes = catModes;
   for (int i = 0; i < _nums; i++) _numMeans[i] = _adaptedFrame.vec(_cats + i).mean();
 }