Пример #1
0
  @Test
  public void test2() throws Exception {
    final IndexSegmentImpl heapSegment =
        (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR, ReadMode.heap);
    final IndexSegmentImpl mmapSegment =
        (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR, ReadMode.mmap);

    final Map<String, ColumnMetadata> metadataMap =
        ((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap();
    for (final String column : metadataMap.keySet()) {
      final ImmutableDictionaryReader heapDictionary = heapSegment.getDictionaryFor(column);
      final ImmutableDictionaryReader mmapDictionary = mmapSegment.getDictionaryFor(column);

      final Set<Object> uniques = uniqueEntries.get(column);
      final List<Object> list = Arrays.asList(uniques.toArray());
      Collections.shuffle(list);
      for (final Object entry : list) {
        Assert.assertEquals(mmapDictionary.indexOf(entry), heapDictionary.indexOf(entry));
        if (!column.equals("pageKey")) {
          Assert.assertEquals(false, heapDictionary.indexOf(entry) < 0);
          Assert.assertEquals(false, mmapDictionary.indexOf(entry) < 0);
        }
      }
    }
  }
Пример #2
0
  private void setupSegmentList(int numberOfSegments) throws Exception {
    final String filePath =
        TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA));
    _indexSegmentList.clear();
    if (INDEXES_DIR.exists()) {
      FileUtils.deleteQuietly(INDEXES_DIR);
    }
    INDEXES_DIR.mkdir();

    for (int i = 0; i < numberOfSegments; ++i) {
      final File segmentDir = new File(INDEXES_DIR, "segment_" + i);

      final SegmentGeneratorConfig config =
          SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
              new File(filePath), segmentDir, "dim1", TimeUnit.DAYS, "test");

      final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
      driver.init(config);
      driver.build();

      LOGGER.debug("built at: {}", segmentDir.getAbsolutePath());
      _indexSegmentList.add(
          ColumnarSegmentLoader.load(new File(segmentDir, driver.getSegmentName()), ReadMode.mmap));
    }
  }
Пример #3
0
  @BeforeClass
  public void setup() throws Exception {
    CONFIG_BUILDER = new TestingServerPropertiesBuilder("testTable");

    setupSegmentFor("testTable");
    setUpTestQueries("testTable");

    final PropertiesConfiguration serverConf = CONFIG_BUILDER.build();
    serverConf.setDelimiterParsingDisabled(false);

    final FileBasedInstanceDataManager instanceDataManager =
        FileBasedInstanceDataManager.getInstanceDataManager();
    instanceDataManager.init(
        new FileBasedInstanceDataManagerConfig(serverConf.subset("pinot.server.instance")));
    instanceDataManager.start();

    System.out.println(
        "************************** : " + new File(INDEX_DIR, "segment").getAbsolutePath());
    File segmentFile = new File(INDEX_DIR, "segment").listFiles()[0];
    segmentName = segmentFile.getName();
    final IndexSegment indexSegment = ColumnarSegmentLoader.load(segmentFile, ReadMode.heap);
    instanceDataManager.getTableDataManager("testTable");
    instanceDataManager.getTableDataManager("testTable").addSegment(indexSegment);

    QUERY_EXECUTOR = new ServerQueryExecutorV1Impl(false);
    QUERY_EXECUTOR.init(
        serverConf.subset("pinot.server.query.executor"),
        instanceDataManager,
        new ServerMetrics(new MetricsRegistry()));
  }
Пример #4
0
  private void setupSegmentList(int numberOfSegments) throws Exception {
    final String filePath =
        TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA));
    _indexSegmentList.clear();
    if (INDEXES_DIR.exists()) {
      FileUtils.deleteQuietly(INDEXES_DIR);
    }
    INDEXES_DIR.mkdir();

    for (int i = 0; i < numberOfSegments; ++i) {
      final File segmentDir = new File(INDEXES_DIR, "segment_" + i);

      final SegmentGeneratorConfig config =
          SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
              new File(filePath), segmentDir, "dim" + i, TimeUnit.DAYS, "midas");
      config.setSegmentNamePostfix(String.valueOf(i));
      final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
      driver.init(config);
      driver.build();

      File parent = new File(INDEXES_DIR, "segment_" + String.valueOf(i));
      String segmentName = parent.list()[0];
      _indexSegmentList.add(
          ColumnarSegmentLoader.load(new File(parent, segmentName), ReadMode.mmap));

      System.out.println("built at : " + segmentDir.getAbsolutePath());
    }
  }
Пример #5
0
  @Test
  public void test1() throws Exception {
    final IndexSegmentImpl heapSegment =
        (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR, ReadMode.heap);
    final IndexSegmentImpl mmapSegment =
        (IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR, ReadMode.mmap);

    for (final String column :
        ((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap().keySet()) {
      final ImmutableDictionaryReader heapDictionary = heapSegment.getDictionaryFor(column);
      final ImmutableDictionaryReader mmapDictionary = mmapSegment.getDictionaryFor(column);

      switch (((SegmentMetadataImpl) mmapSegment.getSegmentMetadata())
          .getColumnMetadataMap()
          .get(column)
          .getDataType()) {
        case BOOLEAN:
        case STRING:
          Assert.assertEquals(true, heapDictionary instanceof StringDictionary);
          Assert.assertEquals(true, mmapDictionary instanceof StringDictionary);
          break;
        case DOUBLE:
          Assert.assertEquals(true, heapDictionary instanceof DoubleDictionary);
          Assert.assertEquals(true, mmapDictionary instanceof DoubleDictionary);
          break;
        case FLOAT:
          Assert.assertEquals(true, heapDictionary instanceof FloatDictionary);
          Assert.assertEquals(true, mmapDictionary instanceof FloatDictionary);
          break;
        case LONG:
          Assert.assertEquals(true, heapDictionary instanceof LongDictionary);
          Assert.assertEquals(true, mmapDictionary instanceof LongDictionary);
          break;
        case INT:
          Assert.assertEquals(true, heapDictionary instanceof IntDictionary);
          Assert.assertEquals(true, mmapDictionary instanceof IntDictionary);
          break;
      }

      Assert.assertEquals(mmapDictionary.length(), heapDictionary.length());
      for (int i = 0; i < heapDictionary.length(); i++) {
        Assert.assertEquals(mmapDictionary.get(i), heapDictionary.get(i));
      }
    }
  }
Пример #6
0
  private void setupSegment() throws Exception {
    final String filePath =
        TestUtils.getFileFromResourceUrl(
            PlanMakerTest.class.getClassLoader().getResource(LARGE_AVRO_DATA));

    if (INDEX_DIR.exists()) {
      FileUtils.deleteQuietly(INDEX_DIR);
    }

    final SegmentGeneratorConfig config =
        SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
            new File(filePath), INDEX_DIR, "dim1", TimeUnit.DAYS, "test");

    final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();

    LOGGER.debug("built at: {}", INDEX_DIR.getAbsolutePath());
    final File indexSegmentDir = new File(INDEX_DIR, driver.getSegmentName());
    _indexSegment = ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.mmap);
  }
  private void setupSegment() throws Exception {
    final String filePath =
        TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));

    if (INDEX_DIR.exists()) {
      FileUtils.deleteQuietly(INDEX_DIR);
    }

    final SegmentGeneratorConfig config =
        SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
            new File(filePath), INDEX_DIR, "time_day", TimeUnit.DAYS, "test");

    final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
    driver.init(config);
    driver.build();

    LOGGER.info("built at : {}", INDEX_DIR.getAbsolutePath());
    final File indexSegmentDir = new File(INDEX_DIR, driver.getSegmentName());
    _indexSegment = ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.heap);
    _medataMap =
        ((SegmentMetadataImpl) ((IndexSegmentImpl) _indexSegment).getSegmentMetadata())
            .getColumnMetadataMap();
  }