@Override
 public void compress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
   if (inlength == 0) return;
   int initoffset = 0;
   ByteBuffer buf = ByteBuffer.allocateDirect(inlength * 8);
   for (int k = inpos.get(); k < inpos.get() + inlength; ++k) {
     int val = in[k] - initoffset;
     initoffset = in[k];
     do {
       int b = (val & 127);
       val >>>= 7;
       if (val != 0) {
         b |= 128;
       }
       buf.put((byte) b);
     } while (val != 0);
   }
   while (buf.position() % 4 != 0) buf.put((byte) 128);
   final int length = buf.position();
   buf.flip();
   IntBuffer ibuf = buf.asIntBuffer();
   ibuf.get(out, outpos.get(), length / 4);
   outpos.add(length / 4);
   inpos.add(inlength);
 }
 @Override
 public void uncompress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
   if (inlength == 0) return;
   int s = 0;
   int p = inpos.get();
   int finalp = inpos.get() + inlength;
   int tmpoutpos = outpos.get();
   int initoffset = 0;
   for (int v = 0, shift = 0; p < finalp; ) {
     int c = (byte) (in[p] >>> (24 - s));
     s += 8;
     if (s == 32) {
       s = 0;
       p++;
     }
     v += ((c & 127) << shift);
     if ((c & 128) == 0) {
       out[tmpoutpos] = v + initoffset;
       initoffset = out[tmpoutpos];
       tmpoutpos++;
       v = 0;
       shift = 0;
     } else shift += 7;
   }
   outpos.set(tmpoutpos);
   inpos.add(inlength);
 }
Esempio n. 3
0
  private static void bytebench(ArrayList<int[]> postings, CompressionMode cm, boolean verbose) {
    int maxlength = 0;
    for (int[] x : postings) if (maxlength < x.length) maxlength = x.length;
    if (verbose) System.out.println("Max array length: " + maxlength);
    byte[] compbuffer = new byte[6 * (maxlength + 1024)];
    int[] decompbuffer = new int[maxlength];
    if (verbose) System.out.println("Scheme -- bits/int -- speed (mis)");
    for (ByteIntegerCODEC c : (cm == CompressionMode.DELTA ? bcodecs : regbcodecs)) {
      long bef = 0;
      long aft = 0;
      long decomptime = 0;
      long volumein = 0;
      long volumeout = 0;
      byte[][] compdata = new byte[postings.size()][];
      for (int k = 0; k < postings.size(); ++k) {
        int[] in = postings.get(k);
        IntWrapper inpos = new IntWrapper(0);
        IntWrapper outpos = new IntWrapper(0);
        c.compress(in, inpos, in.length, compbuffer, outpos);
        int clength = outpos.get();
        inpos = new IntWrapper(0);
        outpos = new IntWrapper(0);
        c.uncompress(compbuffer, inpos, clength, decompbuffer, outpos);
        volumein += in.length;
        volumeout += clength;

        if (outpos.get() != in.length) throw new RuntimeException("bug");
        for (int z = 0; z < in.length; ++z)
          if (in[z] != decompbuffer[z]) throw new RuntimeException("bug");
        compdata[k] = Arrays.copyOf(compbuffer, clength);
      }
      bef = System.nanoTime();
      for (byte[] cin : compdata) {
        IntWrapper inpos = new IntWrapper(0);
        IntWrapper outpos = new IntWrapper(0);
        c.uncompress(cin, inpos, cin.length, decompbuffer, outpos);
        if (inpos.get() != cin.length) throw new RuntimeException("bug");
      }
      aft = System.nanoTime();
      decomptime += (aft - bef);
      double bitsPerInt = volumeout * 8.0 / volumein;
      double decompressSpeed = volumein * 1000.0 / (decomptime);
      if (verbose)
        System.out.println(
            c.toString() + "\t" + String.format("\t%1$.2f\t%2$.2f", bitsPerInt, decompressSpeed));
    }
  }
Esempio n. 4
0
  public static SLNode nthToLast2(SLNode head, int k, IntWrapper i) {
    if (head.next == null) return head;

    SLNode node = nthToLast2(head.next, k, i);
    i.value = i.value + 1;
    if (i.value == k) return head;
    return node;
  }
  @Override
  public void compress(int[] in, IntWrapper inpos, int inlength, byte[] out, IntWrapper outpos) {
    if (inlength == 0) return;
    int outp = outpos.get();
    int initoffset = 0;

    for (int k = inpos.get(); k < inpos.get() + inlength; ++k) {
      int val = in[k] - initoffset;
      initoffset = in[k];
      do {
        int b = (val & 127);
        val >>>= 7;
        if (val != 0) {
          b |= 128;
        }
        out[outp++] = (byte) b;
      } while (val != 0);
    }
    outpos.set(outp);
    inpos.add(inlength);
  }
Esempio n. 6
0
  public void uncompress(
      int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos) {
    if (inLen == 0) return;

    final int outLen = inBuf[inPos.get()];
    inPos.increment();

    int context = 0;
    final int[] work = new int[32];

    int ip = inPos.get();
    int op = outPos.get();
    final int outPosLast = op + outLen;
    for (; op < outPosLast; op += BLOCK_LENGTH) {
      final int bits1 = (inBuf[ip] >>> 24);
      final int bits2 = (inBuf[ip] >>> 16) & 0xFF;
      final int bits3 = (inBuf[ip] >>> 8) & 0xFF;
      final int bits4 = (inBuf[ip] >>> 0) & 0xFF;
      ++ip;
      ip += xorUnpack(inBuf, ip, outBuf, op + 0, bits1, context, work);
      ip += xorUnpack(inBuf, ip, outBuf, op + 32, bits2, outBuf[op + 31], work);
      ip += xorUnpack(inBuf, ip, outBuf, op + 64, bits3, outBuf[op + 63], work);
      ip += xorUnpack(inBuf, ip, outBuf, op + 96, bits4, outBuf[op + 95], work);
      context = outBuf[op + 127];
    }

    outPos.add(outLen);
    inPos.set(ip);
  }
Esempio n. 7
0
  public void compress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos) {
    inLen = inLen - inLen % BLOCK_LENGTH;
    if (inLen == 0) return;

    outBuf[outPos.get()] = inLen;
    outPos.increment();

    int context = 0;
    final int[] work = new int[32];

    int op = outPos.get();
    int ip = inPos.get();
    final int inPosLast = ip + inLen;
    for (; ip < inPosLast; ip += BLOCK_LENGTH) {
      final int bits1 = xorMaxBits(inBuf, ip + 0, 32, context);
      final int bits2 = xorMaxBits(inBuf, ip + 32, 32, inBuf[ip + 31]);
      final int bits3 = xorMaxBits(inBuf, ip + 64, 32, inBuf[ip + 63]);
      final int bits4 = xorMaxBits(inBuf, ip + 96, 32, inBuf[ip + 95]);
      outBuf[op++] = (bits1 << 24) | (bits2 << 16) | (bits3 << 8) | (bits4 << 0);
      op += xorPack(inBuf, ip + 0, outBuf, op, bits1, context, work);
      op += xorPack(inBuf, ip + 32, outBuf, op, bits2, inBuf[ip + 31], work);
      op += xorPack(inBuf, ip + 64, outBuf, op, bits3, inBuf[ip + 63], work);
      op += xorPack(inBuf, ip + 96, outBuf, op, bits4, inBuf[ip + 95], work);
      context = inBuf[ip + 127];
    }

    inPos.add(inLen);
    outPos.set(op);
  }
 @Override
 public void uncompress(byte[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
   int p = inpos.get();
   int finalp = inpos.get() + inlength;
   int tmpoutpos = outpos.get();
   int v = 0;
   int shift = 0;
   int initoffset = 0;
   for (; p < finalp; ++p) {
     byte c = in[p];
     v += ((c & 127) << shift);
     if ((c & 128) == 0) {
       out[tmpoutpos] = v + initoffset;
       initoffset = out[tmpoutpos];
       v = 0;
       ++tmpoutpos;
       shift = 0;
     } else shift += 7;
   }
   outpos.set(tmpoutpos);
   inpos.add(p);
 }
Esempio n. 9
0
 public void compress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
   // Util.assertTrue(inpos.get()+inlength <= in.length);
   System.arraycopy(in, inpos.get(), out, outpos.get(), inlength);
   inpos.add(inlength);
   outpos.add(inlength);
 }