@Override
public void close() {
bitSet.close();
indexDataBuffer.close();
bitSet = null;
indexDataBuffer = null;
}
public FixedBitSingleValueMultiColWriter(File file, int rows, int cols, int[] columnSizesInBits)
throws Exception {
init(rows, cols, columnSizesInBits);
this.indexDataBuffer =
PinotDataBuffer.fromFile(
file,
0,
bytesRequired,
ReadMode.mmap,
FileChannel.MapMode.READ_WRITE,
file.getAbsolutePath() + this.getClass().getCanonicalName());
bitSet = PinotDataCustomBitSet.withDataBuffer(bytesRequired, indexDataBuffer);
}
public void testSingleColMultiValue(
Class<? extends SingleColumnMultiValueWriter> writerClazz,
Class<? extends SingleColumnMultiValueReader> readerClazz)
throws Exception {
LOGGER.info(
"Testing for writerClazz:{} readerClass:{}", writerClazz.getName(), readerClazz.getName());
Constructor<? extends SingleColumnMultiValueWriter> writerClazzConstructor =
writerClazz.getConstructor(new Class[] {File.class, int.class, int.class, int.class});
Constructor<? extends SingleColumnMultiValueReader> readerClazzConstructor =
readerClazz.getConstructor(
new Class[] {PinotDataBuffer.class, int.class, int.class, int.class, boolean.class});
int maxBits = 1;
while (maxBits < 32) {
final String fileName = getClass().getName() + "_test_single_col_mv_fixed_bit.dat";
final File f = new File(fileName);
f.delete();
int numDocs = 10;
int maxNumValues = 100;
final int[][] data = new int[numDocs][];
final Random r = new Random();
final int maxValue = (int) Math.pow(2, maxBits);
int totalNumValues = 0;
int[] startOffsets = new int[numDocs];
int[] lengths = new int[numDocs];
for (int i = 0; i < data.length; i++) {
final int numValues = r.nextInt(maxNumValues) + 1;
data[i] = new int[numValues];
for (int j = 0; j < numValues; j++) {
data[i][j] = r.nextInt(maxValue);
}
startOffsets[i] = totalNumValues;
lengths[i] = numValues;
totalNumValues = totalNumValues + numValues;
}
SingleColumnMultiValueWriter writer =
writerClazzConstructor.newInstance(new Object[] {f, numDocs, totalNumValues, maxBits});
for (int i = 0; i < data.length; i++) {
writer.setIntArray(i, data[i]);
}
writer.close();
final RandomAccessFile raf = new RandomAccessFile(f, "rw");
raf.close();
// Test heap mode
PinotDataBuffer heapBuffer =
PinotDataBuffer.fromFile(f, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
SingleColumnMultiValueReader<? extends ReaderContext> heapReader =
readerClazzConstructor.newInstance(
new Object[] {heapBuffer, numDocs, totalNumValues, maxBits, false});
final int[] readValues = new int[maxNumValues];
for (int i = 0; i < data.length; i++) {
final int numValues = heapReader.getIntArray(i, readValues);
Assert.assertEquals(numValues, data[i].length);
for (int j = 0; j < numValues; j++) {
Assert.assertEquals(readValues[j], data[i][j]);
}
}
heapReader.close();
heapBuffer.close();
// Test mmap mode
PinotDataBuffer mmapBuffer =
PinotDataBuffer.fromFile(f, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
SingleColumnMultiValueReader<? extends ReaderContext> mmapReader =
readerClazzConstructor.newInstance(
new Object[] {mmapBuffer, numDocs, totalNumValues, maxBits, false});
for (int i = 0; i < data.length; i++) {
final int numValues = mmapReader.getIntArray(i, readValues);
Assert.assertEquals(numValues, data[i].length);
for (int j = 0; j < numValues; j++) {
Assert.assertEquals(readValues[j], data[i][j]);
}
}
// Assert.assertEquals(FileReaderTestUtils.getNumOpenFiles(f), 2);
mmapReader.close();
mmapBuffer.close();
// Assert.assertEquals(FileReaderTestUtils.getNumOpenFiles(f), 0);
f.delete();
maxBits = maxBits + 1;
}
LOGGER.info(
"DONE: Testing for writerClazz:{} readerClass:{}",
writerClazz.getName(),
readerClazz.getName());
}