public static CompressedLongsIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == version) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); final CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.forId(buffer.get()); return new CompressedLongsIndexedSupplier( totalSize, sizePer, GenericIndexed.read( buffer, CompressedLongBufferObjectStrategy.getBufferForOrder(order, compression, sizePer)), compression); } else if (versionFromBuffer == LZF_VERSION) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); final CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.LZF; return new CompressedLongsIndexedSupplier( totalSize, sizePer, GenericIndexed.read( buffer, CompressedLongBufferObjectStrategy.getBufferForOrder(order, compression, sizePer)), compression); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
public static CompressedVSizeIntsIndexedSupplier fromByteBuffer( ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == version) { final int numBytes = buffer.get(); final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); final int chunkBytes = sizePer * numBytes + bufferPadding(numBytes); final CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.forId(buffer.get()); return new CompressedVSizeIntsIndexedSupplier( totalSize, sizePer, numBytes, GenericIndexed.read( buffer, CompressedByteBufferObjectStrategy.getBufferForOrder(order, compression, chunkBytes)), compression); } throw new IAE("Unknown version[%s]", versionFromBuffer); }
public CompressedLongsIndexedSupplier convertByteOrder(ByteOrder order) { return new CompressedLongsIndexedSupplier( totalSize, sizePer, GenericIndexed.fromIterable( baseLongBuffers, CompressedLongBufferObjectStrategy.getBufferForOrder(order, compression, sizePer)), compression); }
public static CompressedLongsIndexedSupplier fromList( final List<Long> list, final int chunkFactor, final ByteOrder byteOrder, CompressedObjectStrategy.CompressionStrategy compression) { Preconditions.checkArgument( chunkFactor <= MAX_LONGS_IN_BUFFER, "Chunks must be <= 64k bytes. chunkFactor was[%s]", chunkFactor); return new CompressedLongsIndexedSupplier( list.size(), chunkFactor, GenericIndexed.fromIterable( new Iterable<ResourceHolder<LongBuffer>>() { @Override public Iterator<ResourceHolder<LongBuffer>> iterator() { return new Iterator<ResourceHolder<LongBuffer>>() { int position = 0; @Override public boolean hasNext() { return position < list.size(); } @Override public ResourceHolder<LongBuffer> next() { LongBuffer retVal = LongBuffer.allocate(chunkFactor); if (chunkFactor > list.size() - position) { retVal.limit(list.size() - position); } final List<Long> longs = list.subList(position, position + retVal.remaining()); for (long value : longs) { retVal.put(value); } retVal.rewind(); position += retVal.remaining(); return StupidResourceHolder.create(retVal); } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }, CompressedLongBufferObjectStrategy.getBufferForOrder( byteOrder, compression, chunkFactor)), compression); }
public static CompressedLongsIndexedSupplier fromLongBuffer( final LongBuffer buffer, final int chunkFactor, final ByteOrder byteOrder, CompressedObjectStrategy.CompressionStrategy compression) { Preconditions.checkArgument( chunkFactor <= MAX_LONGS_IN_BUFFER, "Chunks must be <= 64k bytes. chunkFactor was[%s]", chunkFactor); return new CompressedLongsIndexedSupplier( buffer.remaining(), chunkFactor, GenericIndexed.fromIterable( new Iterable<ResourceHolder<LongBuffer>>() { @Override public Iterator<ResourceHolder<LongBuffer>> iterator() { return new Iterator<ResourceHolder<LongBuffer>>() { LongBuffer myBuffer = buffer.asReadOnlyBuffer(); @Override public boolean hasNext() { return myBuffer.hasRemaining(); } @Override public ResourceHolder<LongBuffer> next() { LongBuffer retVal = myBuffer.asReadOnlyBuffer(); if (chunkFactor < myBuffer.remaining()) { retVal.limit(retVal.position() + chunkFactor); } myBuffer.position(myBuffer.position() + retVal.remaining()); return StupidResourceHolder.create(retVal); } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }, CompressedLongBufferObjectStrategy.getBufferForOrder( byteOrder, compression, chunkFactor)), compression); }
public static CompressedVSizeIntsIndexedSupplier fromList( final List<Integer> list, final int maxValue, final int chunkFactor, final ByteOrder byteOrder, CompressedObjectStrategy.CompressionStrategy compression) { final int numBytes = VSizeIndexedInts.getNumBytesForMax(maxValue); final int chunkBytes = chunkFactor * numBytes + bufferPadding(numBytes); Preconditions.checkArgument( chunkFactor <= maxIntsInBufferForBytes(numBytes), "Chunks must be <= 64k bytes. chunkFactor was[%s]", chunkFactor); return new CompressedVSizeIntsIndexedSupplier( list.size(), chunkFactor, numBytes, GenericIndexed.fromIterable( new Iterable<ResourceHolder<ByteBuffer>>() { @Override public Iterator<ResourceHolder<ByteBuffer>> iterator() { return new Iterator<ResourceHolder<ByteBuffer>>() { int position = 0; @Override public boolean hasNext() { return position < list.size(); } @Override public ResourceHolder<ByteBuffer> next() { ByteBuffer retVal = ByteBuffer.allocate(chunkBytes).order(byteOrder); if (chunkFactor > list.size() - position) { retVal.limit((list.size() - position) * numBytes); } else { retVal.limit(chunkFactor * numBytes); } final List<Integer> ints = list.subList(position, position + retVal.remaining() / numBytes); final ByteBuffer buf = ByteBuffer.allocate(Ints.BYTES).order(byteOrder); final boolean bigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN); for (int value : ints) { buf.putInt(0, value); if (bigEndian) { retVal.put(buf.array(), Ints.BYTES - numBytes, numBytes); } else { retVal.put(buf.array(), 0, numBytes); } } retVal.rewind(); position += retVal.remaining() / numBytes; return StupidResourceHolder.create(retVal); } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }, CompressedByteBufferObjectStrategy.getBufferForOrder( byteOrder, compression, chunkBytes)), compression); }