public void produce(SimplifiedLog message, String topicName) {
if (producer == null)
producer =
KafkaUtils.createProducer(
KafkaUtils.createZkClient(zkServer.getConnectString()), ASYNC, false);
producer.send(new KeyedMessage<>(topicName, message.getHostName(), message));
LOGGER.debug("Sent message: {}", message);
}
public void start() throws Exception {
zkServer = new TestingServer(zkPort, true);
zkClient = KafkaUtils.createZkClient(zkServer.getConnectString());
kafkaServer = new KafkaServerStartable(new KafkaConfig(getServerProperties()));
kafkaServer.startup();
LOGGER.debug("Started Kafka server at port {}", kafkaPort);
}
public List<SimplifiedLog> consume(String topicName, int topicPartitions, int expectedMsg)
throws InterruptedException {
ConsumerConnector consumer =
KafkaUtils.createConsumer(zkServer.getConnectString(), "test_group", "1");
List<KafkaStream<String, SimplifiedLog>> streams =
KafkaUtils.getConsumerStreams(consumer, topicName, topicPartitions);
List<Callable<List<SimplifiedLog>>> tasks = new ArrayList<>();
streams.forEach(stream -> tasks.add(createConsumerThread(stream.iterator(), expectedMsg)));
ExecutorService executor = Executors.newFixedThreadPool(streams.size());
List<Future<List<SimplifiedLog>>> futures =
executor.invokeAll(tasks, 5 * expectedMsg, TimeUnit.SECONDS);
List<SimplifiedLog> received = getResultsFromFutures(futures);
consumer.shutdown();
return received;
}