@Override
 public void close() {
   bitSet.close();
   indexDataBuffer.close();
   bitSet = null;
   indexDataBuffer = null;
 }
 public FixedBitSingleValueMultiColWriter(File file, int rows, int cols, int[] columnSizesInBits)
     throws Exception {
   init(rows, cols, columnSizesInBits);
   this.indexDataBuffer =
       PinotDataBuffer.fromFile(
           file,
           0,
           bytesRequired,
           ReadMode.mmap,
           FileChannel.MapMode.READ_WRITE,
           file.getAbsolutePath() + this.getClass().getCanonicalName());
   bitSet = PinotDataCustomBitSet.withDataBuffer(bytesRequired, indexDataBuffer);
 }
Esempio n. 3
0
  public void testSingleColMultiValue(
      Class<? extends SingleColumnMultiValueWriter> writerClazz,
      Class<? extends SingleColumnMultiValueReader> readerClazz)
      throws Exception {
    LOGGER.info(
        "Testing for writerClazz:{} readerClass:{}", writerClazz.getName(), readerClazz.getName());
    Constructor<? extends SingleColumnMultiValueWriter> writerClazzConstructor =
        writerClazz.getConstructor(new Class[] {File.class, int.class, int.class, int.class});
    Constructor<? extends SingleColumnMultiValueReader> readerClazzConstructor =
        readerClazz.getConstructor(
            new Class[] {PinotDataBuffer.class, int.class, int.class, int.class, boolean.class});
    int maxBits = 1;
    while (maxBits < 32) {
      final String fileName = getClass().getName() + "_test_single_col_mv_fixed_bit.dat";
      final File f = new File(fileName);
      f.delete();
      int numDocs = 10;
      int maxNumValues = 100;
      final int[][] data = new int[numDocs][];
      final Random r = new Random();
      final int maxValue = (int) Math.pow(2, maxBits);
      int totalNumValues = 0;
      int[] startOffsets = new int[numDocs];
      int[] lengths = new int[numDocs];
      for (int i = 0; i < data.length; i++) {
        final int numValues = r.nextInt(maxNumValues) + 1;
        data[i] = new int[numValues];
        for (int j = 0; j < numValues; j++) {
          data[i][j] = r.nextInt(maxValue);
        }
        startOffsets[i] = totalNumValues;
        lengths[i] = numValues;
        totalNumValues = totalNumValues + numValues;
      }

      SingleColumnMultiValueWriter writer =
          writerClazzConstructor.newInstance(new Object[] {f, numDocs, totalNumValues, maxBits});

      for (int i = 0; i < data.length; i++) {
        writer.setIntArray(i, data[i]);
      }
      writer.close();

      final RandomAccessFile raf = new RandomAccessFile(f, "rw");
      raf.close();

      // Test heap mode
      PinotDataBuffer heapBuffer =
          PinotDataBuffer.fromFile(f, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
      SingleColumnMultiValueReader<? extends ReaderContext> heapReader =
          readerClazzConstructor.newInstance(
              new Object[] {heapBuffer, numDocs, totalNumValues, maxBits, false});
      final int[] readValues = new int[maxNumValues];
      for (int i = 0; i < data.length; i++) {
        final int numValues = heapReader.getIntArray(i, readValues);
        Assert.assertEquals(numValues, data[i].length);
        for (int j = 0; j < numValues; j++) {
          Assert.assertEquals(readValues[j], data[i][j]);
        }
      }
      heapReader.close();
      heapBuffer.close();

      // Test mmap mode
      PinotDataBuffer mmapBuffer =
          PinotDataBuffer.fromFile(f, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");

      SingleColumnMultiValueReader<? extends ReaderContext> mmapReader =
          readerClazzConstructor.newInstance(
              new Object[] {mmapBuffer, numDocs, totalNumValues, maxBits, false});
      for (int i = 0; i < data.length; i++) {
        final int numValues = mmapReader.getIntArray(i, readValues);
        Assert.assertEquals(numValues, data[i].length);
        for (int j = 0; j < numValues; j++) {
          Assert.assertEquals(readValues[j], data[i][j]);
        }
      }
      // Assert.assertEquals(FileReaderTestUtils.getNumOpenFiles(f), 2);
      mmapReader.close();
      mmapBuffer.close();
      // Assert.assertEquals(FileReaderTestUtils.getNumOpenFiles(f), 0);

      f.delete();
      maxBits = maxBits + 1;
    }
    LOGGER.info(
        "DONE: Testing for writerClazz:{} readerClass:{}",
        writerClazz.getName(),
        readerClazz.getName());
  }