private int computeLength(long rowOffSetStart) {
long rowOffSetEnd = customBitSet.nextSetBitAfter(rowOffSetStart);
if (rowOffSetEnd < 0) {
return (int) (totalNumValues - rowOffSetStart);
}
return (int) (rowOffSetEnd - rowOffSetStart);
}
/**
* @param row
* @param col
* @return
*/
public int getInt(int row, int col) {
final long startBitOffset = computeBitOffset(row, col);
final long endBitOffset = startBitOffset + colSizesInBits[col];
int ret = customBitSet.readInt(startBitOffset, endBitOffset);
ret = ret - offsets[col];
return ret;
}
private int computeStartOffset(int row) {
int chunkId = row / docsPerChunk;
int chunkIdOffset = chunkOffsetsReader.getInt(chunkId, 0);
if (row % docsPerChunk == 0) {
return chunkIdOffset;
}
return (int) customBitSet.findNthBitSetAfter(chunkIdOffset, row - chunkId * docsPerChunk);
}
/**
* @param buffer
* @param rows
* @param cols
* @param columnSizesInBits in bytes
* @param signed offset to each element to make it non negative
* @throws IOException
*/
private FixedBitWidthRowColDataFileReader(
ByteBuffer buffer, int rows, int cols, int[] columnSizesInBits, boolean[] signed)
throws IOException {
this.byteBuffer = buffer;
ownsByteBuffer = false;
this.isMmap = false;
init(rows, cols, columnSizesInBits, signed);
customBitSet = CustomBitSet.withByteBuffer(totalSizeInBytes, byteBuffer);
}
/**
* @param dataFile
* @param rows
* @param cols
* @param columnSizesInBits in bytes
* @param signed , true if the data consists of negative numbers
* @param isMmap heap or mmmap
* @throws IOException
*/
private FixedBitWidthRowColDataFileReader(
File dataFile, int rows, int cols, int[] columnSizesInBits, boolean[] signed, boolean isMmap)
throws IOException {
init(rows, cols, columnSizesInBits, signed);
file = new RandomAccessFile(dataFile, "rw");
this.isMmap = isMmap;
if (isMmap) {
byteBuffer = file.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalSizeInBytes).load();
} else {
byteBuffer = ByteBuffer.allocateDirect(totalSizeInBytes);
file.getChannel().read(byteBuffer);
file.close();
}
ownsByteBuffer = true;
customBitSet = CustomBitSet.withByteBuffer(totalSizeInBytes, byteBuffer);
}
public FixedBitSkipListSCMVReader(
File file,
int numDocs,
int totalNumValues,
int columnSizeInBits,
boolean signed,
boolean isMmap)
throws Exception {
this.numDocs = numDocs;
this.totalNumValues = totalNumValues;
float averageValuesPerDoc = totalNumValues / numDocs;
this.docsPerChunk = (int) (Math.ceil(PREFERRED_NUM_VALUES_PER_CHUNK / averageValuesPerDoc));
this.numChunks = (numDocs + docsPerChunk - 1) / docsPerChunk;
chunkOffsetHeaderSize = numChunks * SIZE_OF_INT * NUM_COLS_IN_HEADER;
bitsetSize = (totalNumValues + 7) / 8;
rawDataSize = Ints.checkedCast(((long) totalNumValues * columnSizeInBits + 7) / 8);
totalSize = chunkOffsetHeaderSize + bitsetSize + rawDataSize;
raf = new RandomAccessFile(file, "rw");
this.isMmap = isMmap;
if (isMmap) {
chunkOffsetsBuffer =
raf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, chunkOffsetHeaderSize).load();
bitsetBuffer =
raf.getChannel()
.map(FileChannel.MapMode.READ_WRITE, chunkOffsetHeaderSize, bitsetSize)
.load();
rawDataBuffer =
raf.getChannel()
.map(FileChannel.MapMode.READ_WRITE, chunkOffsetHeaderSize + bitsetSize, rawDataSize)
.load();
chunkOffsetsReader =
new FixedByteWidthRowColDataFileReader(
chunkOffsetsBuffer, numDocs, NUM_COLS_IN_HEADER, new int[] {SIZE_OF_INT});
customBitSet = CustomBitSet.withByteBuffer(bitsetSize, bitsetBuffer);
rawDataReader =
FixedBitWidthRowColDataFileReader.forByteBuffer(
rawDataBuffer,
totalNumValues,
1,
new int[] {columnSizeInBits},
new boolean[] {signed});
} else {
chunkOffsetsBuffer = ByteBuffer.allocateDirect(chunkOffsetHeaderSize);
raf.getChannel().read(chunkOffsetsBuffer);
chunkOffsetsReader =
new FixedByteWidthRowColDataFileReader(
chunkOffsetsBuffer, numDocs, NUM_COLS_IN_HEADER, new int[] {SIZE_OF_INT});
bitsetBuffer = ByteBuffer.allocateDirect(bitsetSize);
raf.getChannel().read(bitsetBuffer);
customBitSet = CustomBitSet.withByteBuffer(bitsetSize, bitsetBuffer);
rawDataBuffer = ByteBuffer.allocateDirect(rawDataSize);
raf.getChannel().read(rawDataBuffer);
rawDataReader =
FixedBitWidthRowColDataFileReader.forByteBuffer(
rawDataBuffer,
totalNumValues,
1,
new int[] {columnSizeInBits},
new boolean[] {signed});
raf.close();
}
}