Пример #1
0
 // Slow-path append data
 private void append2slow() {
   if (_len > Vec.CHUNK_SZ) throw new ArrayIndexOutOfBoundsException(_len);
   assert _ds == null;
   if (_len2 == _len) { // Check for sparse-ness now & then
     int nzcnt = 0;
     for (int i = 0; i < _len; i++) {
       if (_ls[i] != 0) nzcnt++;
       if (_xs[i] != 0) {
         nzcnt = Vec.CHUNK_SZ;
         break;
       } // Only non-specials sparse
     }
     if (_len >= 32 && nzcnt * 8 <= _len) { // Heuristic for sparseness
       _len = 0;
       for (int i = 0; i < _len2; i++)
         if (_ls[i] != 0) {
           _xs[_len] = i; // Row number in xs
           _ls[_len++] = _ls[i]; // Sparse value in ls
         }
       return; // Compressed, so lots of room now
     }
   }
   _xs = _ls == null ? MemoryManager.malloc4(4) : MemoryManager.arrayCopyOf(_xs, _len << 1);
   _ls = _ls == null ? MemoryManager.malloc8(4) : MemoryManager.arrayCopyOf(_ls, _len << 1);
 }
Пример #2
0
  // Slow-path append string
  private void append2slowstr() {
    // In case of all NAs and then a string, convert NAs to string NAs
    if (_xs != null) {
      _xs = null;
      _ls = null;
      alloc_str_indices(sparseLen());
      Arrays.fill(_is, -1);
    }

    if (_is != null && _is.length > 0) {
      // Check for sparseness
      if (_id == null) {
        int nzs = 0; // assume one non-null for the element currently being stored
        for (int i : _is) if (i != -1) ++nzs;
        if ((nzs + 1) * _sparseRatio < _len) set_sparse(nzs);
      } else {
        if ((_sparseRatio * (_sparseLen) >> 1) > _len) cancel_sparse();
        else _id = MemoryManager.arrayCopyOf(_id, _sparseLen << 1);
      }

      _is = MemoryManager.arrayCopyOf(_is, sparseLen() << 1);
      /* initialize the memory extension with -1s */
      for (int i = sparseLen(); i < _is.length; i++) _is[i] = -1;
    } else {
      _is = MemoryManager.malloc4(4);
      /* initialize everything with -1s */
      for (int i = 0; i < _is.length; i++) _is[i] = -1;
      if (sparse()) alloc_indices(4);
    }
    assert sparseLen() == 0 || _is.length > sparseLen()
        : "_ls.length = " + _is.length + ", _len = " + sparseLen();
  }
Пример #3
0
 // Slow-path append data
 private void append2slowd() {
   assert _ls == null;
   if (_ds != null && _ds.length > 0) {
     if (_id == null) { // check for sparseness
       int nzs = 0; // assume one non-zero for the element currently being stored
       for (double d : _ds) if (d != 0) ++nzs;
       if ((nzs + 1) * _sparseRatio < _len) set_sparse(nzs);
     } else _id = MemoryManager.arrayCopyOf(_id, sparseLen() << 1);
     _ds = MemoryManager.arrayCopyOf(_ds, sparseLen() << 1);
   } else {
     alloc_doubles(4);
     if (sparse()) alloc_indices(4);
   }
   assert sparseLen() == 0 || _ds.length > sparseLen()
       : "_ds.length = " + _ds.length + ", _len = " + sparseLen();
 }
Пример #4
0
  // Append all of 'nc' onto the current NewChunk.  Kill nc.
  public void add(NewChunk nc) {
    assert _cidx >= 0;
    assert sparseLen() <= _len;
    assert nc.sparseLen() <= nc._len : "_len = " + nc.sparseLen() + ", _len2 = " + nc._len;
    if (nc._len == 0) return;
    if (_len == 0) {
      _ls = nc._ls;
      nc._ls = null;
      _xs = nc._xs;
      nc._xs = null;
      _id = nc._id;
      nc._id = null;
      _ds = nc._ds;
      nc._ds = null;
      _is = nc._is;
      nc._is = null;
      _ss = nc._ss;
      nc._ss = null;
      set_sparseLen(nc.sparseLen());
      set_len(nc._len);
      return;
    }
    if (nc.sparse() != sparse()) { // for now, just make it dense
      cancel_sparse();
      nc.cancel_sparse();
    }
    if (_ds != null) throw H2O.fail();
    while (sparseLen() + nc.sparseLen() >= _xs.length)
      _xs = MemoryManager.arrayCopyOf(_xs, _xs.length << 1);
    _ls = MemoryManager.arrayCopyOf(_ls, _xs.length);
    System.arraycopy(nc._ls, 0, _ls, sparseLen(), nc.sparseLen());
    System.arraycopy(nc._xs, 0, _xs, sparseLen(), nc.sparseLen());
    if (_id != null) {
      assert nc._id != null;
      _id = MemoryManager.arrayCopyOf(_id, _xs.length);
      System.arraycopy(nc._id, 0, _id, sparseLen(), nc.sparseLen());
      for (int i = sparseLen(); i < sparseLen() + nc.sparseLen(); ++i) _id[i] += _len;
    } else assert nc._id == null;

    set_sparseLen(sparseLen() + nc.sparseLen());
    set_len(_len + nc._len);
    nc._ls = null;
    nc._xs = null;
    nc._id = null;
    nc.set_sparseLen(nc.set_len(0));
    assert sparseLen() <= _len;
  }
Пример #5
0
 // Slow-path append data
 private void append2slowUUID() {
   if (_ds == null && _ls != null) { // This can happen for columns with all NAs and then a UUID
     _xs = null;
     alloc_doubles(sparseLen());
     Arrays.fill(_ls, C16Chunk._LO_NA);
     Arrays.fill(_ds, Double.longBitsToDouble(C16Chunk._HI_NA));
   }
   if (_ls != null && _ls.length > 0) {
     _ls = MemoryManager.arrayCopyOf(_ls, sparseLen() << 1);
     _ds = MemoryManager.arrayCopyOf(_ds, sparseLen() << 1);
   } else {
     alloc_mantissa(4);
     alloc_doubles(4);
   }
   assert sparseLen() == 0 || _ls.length > sparseLen()
       : "_ls.length = " + _ls.length + ", _len = " + sparseLen();
 }
Пример #6
0
 private void append_ss(String str) {
   if (_ss == null) {
     _ss = MemoryManager.malloc1((str.length() + 1) * 4);
   }
   while (_ss.length < (_sslen + str.length() + 1)) {
     _ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);
   }
   for (byte b : str.getBytes()) _ss[_sslen++] = b;
   _ss[_sslen++] = (byte) 0; // for trailing 0;
 }
Пример #7
0
  // Slow-path append data
  private void append2slow() {
    if (sparseLen() > FileVec.DFLT_CHUNK_SIZE)
      throw new ArrayIndexOutOfBoundsException(sparseLen());

    assert _ds == null;
    if (_ls != null && _ls.length > 0) {
      if (_id == null) { // check for sparseness
        int nzs = 0;
        for (int i = 0; i < _ls.length; ++i) if (_ls[i] != 0 || _xs[i] != 0) ++nzs;
        if ((nzs + 1) * _sparseRatio < _len) {
          set_sparse(nzs);
          assert sparseLen() == 0 || sparseLen() <= _ls.length
              : "_len = "
                  + sparseLen()
                  + ", _ls.length = "
                  + _ls.length
                  + ", nzs = "
                  + nzs
                  + ", len2 = "
                  + _len;
          assert _id.length == _ls.length;
          assert sparseLen() <= _len;
          return;
        }
      } else {
        // verify we're still sufficiently sparse
        if ((_sparseRatio * (sparseLen()) >> 1) > _len) cancel_sparse();
        else _id = MemoryManager.arrayCopyOf(_id, sparseLen() << 1);
      }
      _ls = MemoryManager.arrayCopyOf(_ls, sparseLen() << 1);
      _xs = MemoryManager.arrayCopyOf(_xs, sparseLen() << 1);
    } else {
      alloc_mantissa(4);
      alloc_exponent(4);
      if (_id != null) alloc_indices(4);
    }
    assert sparseLen() == 0 || sparseLen() < _ls.length
        : "_len = " + sparseLen() + ", _ls.length = " + _ls.length;
    assert _id == null || _id.length == _ls.length;
    assert sparseLen() <= _len;
  }
Пример #8
0
  private void append_ss(String str) {
    byte[] bytes = str.getBytes(Charsets.UTF_8);

    // Allocate memory if necessary
    if (_ss == null) _ss = MemoryManager.malloc1((bytes.length + 1) * 4);
    while (_ss.length < (_sslen + bytes.length + 1))
      _ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);

    // Copy bytes to _ss
    for (byte b : bytes) _ss[_sslen++] = b;
    _ss[_sslen++] = (byte) 0; // for trailing 0;
  }
Пример #9
0
  private void append_ss(BufferedString str) {
    int strlen = str.length();
    int off = str.getOffset();
    byte b[] = str.getBuffer();

    if (_ss == null) {
      _ss = MemoryManager.malloc1((strlen + 1) * 4);
    }
    while (_ss.length < (_sslen + strlen + 1)) {
      _ss = MemoryManager.arrayCopyOf(_ss, _ss.length << 1);
    }
    for (int i = off; i < off + strlen; i++) _ss[_sslen++] = b[i];
    _ss[_sslen++] = (byte) 0; // for trailing 0;
  }
Пример #10
0
 public double[] denormalizeBeta(double[] beta) {
   if (beta.length != fullN() + 1) System.out.println("haha");
   assert beta.length == fullN() + 1 : "beta len = " + beta.length;
   beta = MemoryManager.arrayCopyOf(beta, beta.length);
   if (_predictor_transform == DataInfo.TransformType.STANDARDIZE) {
     double norm = 0.0; // Reverse any normalization on the intercept
     // denormalize only the numeric coefs (categoricals are not normalized)
     final int numoff = numStart();
     for (int i = numoff; i < beta.length - 1; i++) {
       double b = beta[i] * _normMul[i - numoff];
       norm += b * _normSub[i - numoff]; // Also accumulate the intercept adjustment
       beta[i] = b;
     }
     beta[beta.length - 1] -= norm;
   }
   return beta;
 }
Пример #11
0
 public double[] denormalizeBeta(double[] beta) {
   int N = fullN() + 1;
   assert (beta.length % N) == 0 : "beta len = " + beta.length + " expected multiple of" + N;
   int nclasses = beta.length / N;
   beta = MemoryManager.arrayCopyOf(beta, beta.length);
   if (_predictor_transform == DataInfo.TransformType.STANDARDIZE) {
     for (int c = 0; c < nclasses; ++c) {
       int off = N * c;
       double norm = 0.0; // Reverse any normalization on the intercept
       // denormalize only the numeric coefs (categoricals are not normalized)
       final int numoff = numStart();
       for (int i = numoff; i < N - 1; i++) {
         if (isInteractionVec(i)) continue; // interactions are not normalized!
         double b = beta[off + i] * _normMul[i - numoff];
         norm += b * _normSub[i - numoff]; // Also accumulate the intercept adjustment
         beta[off + i] = b;
       }
       beta[off + N - 1] -= norm;
     }
   }
   return beta;
 }
Пример #12
0
 // Slow-path append data
 private void append2slowd() {
   if (_len > Vec.CHUNK_SZ) throw new ArrayIndexOutOfBoundsException(_len);
   assert _ls == null && _len2 == _len;
   _ds = _ds == null ? MemoryManager.malloc8d(4) : MemoryManager.arrayCopyOf(_ds, _len << 1);
 }