@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
if (inlength == 0) return;
int initoffset = 0;
ByteBuffer buf = ByteBuffer.allocateDirect(inlength * 8);
for (int k = inpos.get(); k < inpos.get() + inlength; ++k) {
int val = in[k] - initoffset;
initoffset = in[k];
do {
int b = (val & 127);
val >>>= 7;
if (val != 0) {
b |= 128;
}
buf.put((byte) b);
} while (val != 0);
}
while (buf.position() % 4 != 0) buf.put((byte) 128);
final int length = buf.position();
buf.flip();
IntBuffer ibuf = buf.asIntBuffer();
ibuf.get(out, outpos.get(), length / 4);
outpos.add(length / 4);
inpos.add(inlength);
}
@Override
public void uncompress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
if (inlength == 0) return;
int s = 0;
int p = inpos.get();
int finalp = inpos.get() + inlength;
int tmpoutpos = outpos.get();
int initoffset = 0;
for (int v = 0, shift = 0; p < finalp; ) {
int c = (byte) (in[p] >>> (24 - s));
s += 8;
if (s == 32) {
s = 0;
p++;
}
v += ((c & 127) << shift);
if ((c & 128) == 0) {
out[tmpoutpos] = v + initoffset;
initoffset = out[tmpoutpos];
tmpoutpos++;
v = 0;
shift = 0;
} else shift += 7;
}
outpos.set(tmpoutpos);
inpos.add(inlength);
}
private static void bytebench(ArrayList<int[]> postings, CompressionMode cm, boolean verbose) {
int maxlength = 0;
for (int[] x : postings) if (maxlength < x.length) maxlength = x.length;
if (verbose) System.out.println("Max array length: " + maxlength);
byte[] compbuffer = new byte[6 * (maxlength + 1024)];
int[] decompbuffer = new int[maxlength];
if (verbose) System.out.println("Scheme -- bits/int -- speed (mis)");
for (ByteIntegerCODEC c : (cm == CompressionMode.DELTA ? bcodecs : regbcodecs)) {
long bef = 0;
long aft = 0;
long decomptime = 0;
long volumein = 0;
long volumeout = 0;
byte[][] compdata = new byte[postings.size()][];
for (int k = 0; k < postings.size(); ++k) {
int[] in = postings.get(k);
IntWrapper inpos = new IntWrapper(0);
IntWrapper outpos = new IntWrapper(0);
c.compress(in, inpos, in.length, compbuffer, outpos);
int clength = outpos.get();
inpos = new IntWrapper(0);
outpos = new IntWrapper(0);
c.uncompress(compbuffer, inpos, clength, decompbuffer, outpos);
volumein += in.length;
volumeout += clength;
if (outpos.get() != in.length) throw new RuntimeException("bug");
for (int z = 0; z < in.length; ++z)
if (in[z] != decompbuffer[z]) throw new RuntimeException("bug");
compdata[k] = Arrays.copyOf(compbuffer, clength);
}
bef = System.nanoTime();
for (byte[] cin : compdata) {
IntWrapper inpos = new IntWrapper(0);
IntWrapper outpos = new IntWrapper(0);
c.uncompress(cin, inpos, cin.length, decompbuffer, outpos);
if (inpos.get() != cin.length) throw new RuntimeException("bug");
}
aft = System.nanoTime();
decomptime += (aft - bef);
double bitsPerInt = volumeout * 8.0 / volumein;
double decompressSpeed = volumein * 1000.0 / (decomptime);
if (verbose)
System.out.println(
c.toString() + "\t" + String.format("\t%1$.2f\t%2$.2f", bitsPerInt, decompressSpeed));
}
}
public static SLNode nthToLast2(SLNode head, int k, IntWrapper i) {
if (head.next == null) return head;
SLNode node = nthToLast2(head.next, k, i);
i.value = i.value + 1;
if (i.value == k) return head;
return node;
}
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, byte[] out, IntWrapper outpos) {
if (inlength == 0) return;
int outp = outpos.get();
int initoffset = 0;
for (int k = inpos.get(); k < inpos.get() + inlength; ++k) {
int val = in[k] - initoffset;
initoffset = in[k];
do {
int b = (val & 127);
val >>>= 7;
if (val != 0) {
b |= 128;
}
out[outp++] = (byte) b;
} while (val != 0);
}
outpos.set(outp);
inpos.add(inlength);
}
public void uncompress(
int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos) {
if (inLen == 0) return;
final int outLen = inBuf[inPos.get()];
inPos.increment();
int context = 0;
final int[] work = new int[32];
int ip = inPos.get();
int op = outPos.get();
final int outPosLast = op + outLen;
for (; op < outPosLast; op += BLOCK_LENGTH) {
final int bits1 = (inBuf[ip] >>> 24);
final int bits2 = (inBuf[ip] >>> 16) & 0xFF;
final int bits3 = (inBuf[ip] >>> 8) & 0xFF;
final int bits4 = (inBuf[ip] >>> 0) & 0xFF;
++ip;
ip += xorUnpack(inBuf, ip, outBuf, op + 0, bits1, context, work);
ip += xorUnpack(inBuf, ip, outBuf, op + 32, bits2, outBuf[op + 31], work);
ip += xorUnpack(inBuf, ip, outBuf, op + 64, bits3, outBuf[op + 63], work);
ip += xorUnpack(inBuf, ip, outBuf, op + 96, bits4, outBuf[op + 95], work);
context = outBuf[op + 127];
}
outPos.add(outLen);
inPos.set(ip);
}
public void compress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos) {
inLen = inLen - inLen % BLOCK_LENGTH;
if (inLen == 0) return;
outBuf[outPos.get()] = inLen;
outPos.increment();
int context = 0;
final int[] work = new int[32];
int op = outPos.get();
int ip = inPos.get();
final int inPosLast = ip + inLen;
for (; ip < inPosLast; ip += BLOCK_LENGTH) {
final int bits1 = xorMaxBits(inBuf, ip + 0, 32, context);
final int bits2 = xorMaxBits(inBuf, ip + 32, 32, inBuf[ip + 31]);
final int bits3 = xorMaxBits(inBuf, ip + 64, 32, inBuf[ip + 63]);
final int bits4 = xorMaxBits(inBuf, ip + 96, 32, inBuf[ip + 95]);
outBuf[op++] = (bits1 << 24) | (bits2 << 16) | (bits3 << 8) | (bits4 << 0);
op += xorPack(inBuf, ip + 0, outBuf, op, bits1, context, work);
op += xorPack(inBuf, ip + 32, outBuf, op, bits2, inBuf[ip + 31], work);
op += xorPack(inBuf, ip + 64, outBuf, op, bits3, inBuf[ip + 63], work);
op += xorPack(inBuf, ip + 96, outBuf, op, bits4, inBuf[ip + 95], work);
context = inBuf[ip + 127];
}
inPos.add(inLen);
outPos.set(op);
}
@Override
public void uncompress(byte[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
int p = inpos.get();
int finalp = inpos.get() + inlength;
int tmpoutpos = outpos.get();
int v = 0;
int shift = 0;
int initoffset = 0;
for (; p < finalp; ++p) {
byte c = in[p];
v += ((c & 127) << shift);
if ((c & 128) == 0) {
out[tmpoutpos] = v + initoffset;
initoffset = out[tmpoutpos];
v = 0;
++tmpoutpos;
shift = 0;
} else shift += 7;
}
outpos.set(tmpoutpos);
inpos.add(p);
}
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out, IntWrapper outpos) {
// Util.assertTrue(inpos.get()+inlength <= in.length);
System.arraycopy(in, inpos.get(), out, outpos.get(), inlength);
inpos.add(inlength);
outpos.add(inlength);
}