コード例 #1
0
  @BeforeClass
  public void setUp() throws Exception {
    // Start ZK and Kafka
    startZk();
    kafkaStarter =
        KafkaStarterUtils.startServer(
            KafkaStarterUtils.DEFAULT_KAFKA_PORT,
            KafkaStarterUtils.DEFAULT_BROKER_ID,
            KafkaStarterUtils.DEFAULT_ZK_STR,
            KafkaStarterUtils.getDefaultKafkaConfiguration());

    // Create Kafka topic
    KafkaStarterUtils.createTopic(KAFKA_TOPIC, KafkaStarterUtils.DEFAULT_ZK_STR);

    // Start the Pinot cluster
    startController();
    startBroker();
    startServer();

    // Unpack data
    final List<File> avroFiles = unpackAvroData(_tmpDir, SEGMENT_COUNT);

    File schemaFile = getSchemaFile();

    // Load data into H2
    ExecutorService executor = Executors.newCachedThreadPool();
    setupH2AndInsertAvro(avroFiles, executor);

    // Initialize query generator
    setupQueryGenerator(avroFiles, executor);

    // Push data into the Kafka topic
    pushAvroIntoKafka(avroFiles, executor, KAFKA_TOPIC);

    // Wait for data push, query generator initialization and H2 load to complete
    executor.shutdown();
    executor.awaitTermination(10, TimeUnit.MINUTES);

    // Create Pinot table
    setUpTable(
        "mytable",
        "DaysSinceEpoch",
        "daysSinceEpoch",
        KafkaStarterUtils.DEFAULT_ZK_STR,
        KAFKA_TOPIC,
        schemaFile,
        avroFiles.get(0));

    // Wait until the Pinot event count matches with the number of events in the Avro files
    long timeInFiveMinutes = System.currentTimeMillis() + 5 * 60 * 1000L;
    Statement statement =
        _connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
    statement.execute("select count(*) from mytable");
    ResultSet rs = statement.getResultSet();
    rs.first();
    int h2RecordCount = rs.getInt(1);
    rs.close();

    waitForRecordCountToStabilizeToExpectedCount(h2RecordCount, timeInFiveMinutes);
  }
コード例 #2
0
 @AfterClass
 public void tearDown() throws Exception {
   stopBroker();
   stopController();
   stopServer();
   KafkaStarterUtils.stopServer(kafkaStarter);
   try {
     stopZk();
   } catch (Exception e) {
     // Swallow ZK Exceptions.
   }
   FileUtils.deleteDirectory(_tmpDir);
 }