@BeforeClass
public void setUp() throws Exception {
// Start ZK and Kafka
startZk();
kafkaStarter =
KafkaStarterUtils.startServer(
KafkaStarterUtils.DEFAULT_KAFKA_PORT,
KafkaStarterUtils.DEFAULT_BROKER_ID,
KafkaStarterUtils.DEFAULT_ZK_STR,
KafkaStarterUtils.getDefaultKafkaConfiguration());
// Create Kafka topic
KafkaStarterUtils.createTopic(KAFKA_TOPIC, KafkaStarterUtils.DEFAULT_ZK_STR);
// Start the Pinot cluster
startController();
startBroker();
startServer();
// Unpack data
final List<File> avroFiles = unpackAvroData(_tmpDir, SEGMENT_COUNT);
File schemaFile = getSchemaFile();
// Load data into H2
ExecutorService executor = Executors.newCachedThreadPool();
setupH2AndInsertAvro(avroFiles, executor);
// Initialize query generator
setupQueryGenerator(avroFiles, executor);
// Push data into the Kafka topic
pushAvroIntoKafka(avroFiles, executor, KAFKA_TOPIC);
// Wait for data push, query generator initialization and H2 load to complete
executor.shutdown();
executor.awaitTermination(10, TimeUnit.MINUTES);
// Create Pinot table
setUpTable(
"mytable",
"DaysSinceEpoch",
"daysSinceEpoch",
KafkaStarterUtils.DEFAULT_ZK_STR,
KAFKA_TOPIC,
schemaFile,
avroFiles.get(0));
// Wait until the Pinot event count matches with the number of events in the Avro files
long timeInFiveMinutes = System.currentTimeMillis() + 5 * 60 * 1000L;
Statement statement =
_connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
statement.execute("select count(*) from mytable");
ResultSet rs = statement.getResultSet();
rs.first();
int h2RecordCount = rs.getInt(1);
rs.close();
waitForRecordCountToStabilizeToExpectedCount(h2RecordCount, timeInFiveMinutes);
}
@AfterClass
public void tearDown() throws Exception {
stopBroker();
stopController();
stopServer();
KafkaStarterUtils.stopServer(kafkaStarter);
try {
stopZk();
} catch (Exception e) {
// Swallow ZK Exceptions.
}
FileUtils.deleteDirectory(_tmpDir);
}