Esempio n. 1
0
 // Compute a compressed double buffer
 private Chunk chunkD() {
   assert _len2 == _len;
   final byte[] bs = MemoryManager.malloc1(_len * 8);
   for (int i = 0; i < _len; ++i)
     UDP.set8d(
         bs,
         8 * i,
         _ds != null
             ? _ds[i]
             : (isNA(i) || isEnum(i)) ? Double.NaN : _ls[i] * DParseTask.pow10(_xs[i]));
   return new C8DChunk(bs);
 }
Esempio n. 2
0
  Chunk compress() {
    // Check for basic mode info: all missing or all strings or mixed stuff
    byte mode = type();
    if (mode == AppendableVec.NA) // ALL NAs, nothing to do
    return new C0DChunk(Double.NaN, _len);
    for (int i = 0; i < _len; i++)
      if (mode == AppendableVec.ENUM && !isEnum(i) || mode == AppendableVec.NUMBER && isEnum(i))
        setNA_impl(i);
    _naCnt = -1;
    type(); // Re-run rollups after dropping all numbers/enums

    // If the data was set8 as doubles, we do a quick check to see if it's
    // plain longs.  If not, we give up and use doubles.
    if (_ds != null) {
      int i = 0;
      for (; i < _len; i++) // Attempt to inject all doubles into longs
      if (!Double.isNaN(_ds[i]) && (double) (long) _ds[i] != _ds[i]) break;
      if (i < _len) return chunkD();
      _ls = new long[_ds.length]; // Else flip to longs
      _xs = new int[_ds.length];
      for (i = 0; i < _len; i++) // Inject all doubles into longs
      if (Double.isNaN(_ds[i])) _xs[i] = Integer.MIN_VALUE;
        else _ls[i] = (long) _ds[i];
      _ds = null;
    }

    // IF (_len2 > _len) THEN Sparse
    // Check for compressed *during appends*.  Here we know:
    // - No specials; _xs[]==0.
    // - No floats; _ds==null
    // - NZ length in _len, actual length in _len2.
    // - Huge ratio between _len2 and _len, and we do NOT want to inflate to
    //   the larger size; we need to keep it all small all the time.
    // - Rows in _xs

    // Data in some fixed-point format, not doubles
    // See if we can sanely normalize all the data to the same fixed-point.
    int xmin = Integer.MAX_VALUE; // min exponent found
    long lemin = 0, lemax = lemin; // min/max at xmin fixed-point
    boolean overflow = false;
    boolean floatOverflow = false;
    boolean first = true;
    double min = _len2 == _len ? Double.MAX_VALUE : 0;
    double max = _len2 == _len ? -Double.MAX_VALUE : 0;

    for (int i = 0; i < _len; i++) {
      if (isNA(i)) continue;
      long l = _ls[i];
      int x = _xs[i];
      if (x == Integer.MIN_VALUE + 1 || _len2 != _len) x = 0; // Replace enum flag with no scaling
      assert l != 0 || x == 0; // Exponent of zero is always zero
      // Compute per-chunk min/max
      double d = l * DParseTask.pow10(x);
      if (d < min) min = d;
      if (d > max) max = d;
      long t; // Remove extra scaling
      while (l != 0 && (t = l / 10) * 10 == l) {
        l = t;
        x++;
      }
      floatOverflow = Math.abs(l) > MAX_FLOAT_MANTISSA;
      if (first) {
        first = false;
        xmin = x;
        lemin = lemax = l;
        continue;
      }
      // Remove any trailing zeros / powers-of-10
      if (overflow || (overflow = (Math.abs(xmin - x)) >= 10)) continue;
      // Track largest/smallest values at xmin scale.  Note overflow.
      if (x < xmin) {
        lemin *= DParseTask.pow10i(xmin - x);
        lemax *= DParseTask.pow10i(xmin - x);
        xmin = x; // Smaller xmin
      }
      // *this* value, as a long scaled at the smallest scale
      long le = l * DParseTask.pow10i(x - xmin);
      if (le < lemin) lemin = le;
      if (le > lemax) lemax = le;
    }
    if (_len2 != _len) { // sparse? compare xmin/lemin/lemax with 0
      lemin = Math.min(0, lemin);
      lemax = Math.max(0, lemax);
    }

    // Constant column?
    if (_naCnt == 0 && min == max) {
      return ((long) min == min) ? new C0LChunk((long) min, _len2) : new C0DChunk(min, _len2);
    }

    // Boolean column?
    if (max == 1 && min == 0 && xmin == 0) {
      if (_nzCnt * 32 < _len2
          && _naCnt == 0
          && _len2 < 65535
          && xmin == 0) // Very sparse? (and not too big?)
      if (_len2 == _len) return new CX0Chunk(_ls, _len2, _nzCnt); // Dense  constructor
        else return new CX0Chunk(_xs, _len2, _len); // Sparse constructor
      int bpv = _strCnt + _naCnt > 0 ? 2 : 1; // Bit-vector
      byte[] cbuf = bufB(bpv);
      return new CBSChunk(cbuf, cbuf[0], cbuf[1]);
    }

    final boolean fpoint = xmin < 0 || min < Long.MIN_VALUE || max > Long.MAX_VALUE;

    // Result column must hold floats?
    // Highly sparse but not a bitvector or constant?
    if (!fpoint
        && (_nzCnt + _naCnt) * 8 < _len2
        && _len2 < 65535
        && xmin == 0
        && // (and not too big?)
        lemin > Short.MIN_VALUE
        && lemax <= Short.MAX_VALUE) // Only handling unbiased shorts here
    if (_len2 == _len) return new CX2Chunk(_ls, _xs, _len2, _nzCnt, _naCnt); // Sparse byte chunk
      else return new CX2Chunk(_ls, _xs, _len2, _len);

    // Exponent scaling: replacing numbers like 1.3 with 13e-1.  '13' fits in a
    // byte and we scale the column by 0.1.  A set of numbers like
    // {1.2,23,0.34} then is normalized to always be represented with 2 digits
    // to the right: {1.20,23.00,0.34} and we scale by 100: {120,2300,34}.
    // This set fits in a 2-byte short.

    // We use exponent-scaling for bytes & shorts only; it's uncommon (and not
    // worth it) for larger numbers.  We need to get the exponents to be
    // uniform, so we scale up the largest lmax by the largest scale we need
    // and if that fits in a byte/short - then it's worth compressing.  Other
    // wise we just flip to a float or double representation.
    if (overflow || fpoint && floatOverflow || -35 > xmin || xmin > 35) return chunkD();
    if (fpoint) {
      if (lemax - lemin < 255) // Fits in scaled biased byte?
      return new C1SChunk(bufX(lemin, xmin, C1SChunk.OFF, 0), (int) lemin, DParseTask.pow10(xmin));
      if (lemax - lemin < 65535) { // we use signed 2B short, add -32k to the bias!
        long bias = 32767 + lemin;
        return new C2SChunk(bufX(bias, xmin, C2SChunk.OFF, 1), (int) bias, DParseTask.pow10(xmin));
      }
      if (lemax - lemin < Integer.MAX_VALUE)
        return new C4SChunk(
            bufX(lemin, xmin, C4SChunk.OFF, 2), (int) lemin, DParseTask.pow10(xmin));
      return chunkD();
    } // else an integer column
    // Compress column into a byte
    if (xmin == 0 && 0 <= lemin && lemax <= 255 && ((_naCnt + _strCnt) == 0))
      return new C1NChunk(bufX(0, 0, C1NChunk.OFF, 0));
    if (lemax - lemin < 255) { // Span fits in a byte?
      if (0 <= min && max < 255) // Span fits in an unbiased byte?
      return new C1Chunk(bufX(0, 0, C1Chunk.OFF, 0));
      return new C1SChunk(bufX(lemin, xmin, C1SChunk.OFF, 0), (int) lemin, DParseTask.pow10i(xmin));
    }

    // Compress column into a short
    if (lemax - lemin < 65535) { // Span fits in a biased short?
      if (xmin == 0
          && Short.MIN_VALUE < lemin
          && lemax <= Short.MAX_VALUE) // Span fits in an unbiased short?
      return new C2Chunk(bufX(0, 0, C2Chunk.OFF, 1));
      int bias = (int) (lemin - (Short.MIN_VALUE + 1));
      return new C2SChunk(bufX(bias, xmin, C2SChunk.OFF, 1), bias, DParseTask.pow10i(xmin));
    }
    // Compress column into ints
    if (Integer.MIN_VALUE < min && max <= Integer.MAX_VALUE) return new C4Chunk(bufX(0, 0, 0, 2));
    return new C8Chunk(bufX(0, 0, 0, 3));
  }