/**
* Test AbstractKafkaSinglePortInputOperator (i.e. an input adapter for Kafka, aka consumer). This
* module receives data from an outside test generator through Kafka message bus and feed that
* data into Malhar streaming platform.
*
* <p>[Generate message and send that to Kafka message bus] ==> [Receive that message through
* Kafka input adapter(i.e. consumer) and send using emitTuples() interface on output port during
* onMessage call]
*
* @throws Exception
*/
public void testKafkaInputOperator(
int sleepTime, final int totalCount, KafkaConsumer consumer, boolean isValid)
throws Exception {
// initial the latch for this test
latch = new CountDownLatch(1);
// Start producer
KafkaTestProducer p = new KafkaTestProducer(TEST_TOPIC);
p.setSendCount(totalCount);
new Thread(p).start();
// Create DAG for testing.
LocalMode lma = LocalMode.newInstance();
DAG dag = lma.getDAG();
// Create KafkaSinglePortStringInputOperator
KafkaSinglePortStringInputOperator node =
dag.addOperator("Kafka message consumer", KafkaSinglePortStringInputOperator.class);
consumer.setTopic(TEST_TOPIC);
if (isValid) {
Set<String> brokerSet = new HashSet<String>();
brokerSet.add("localhost:9092");
consumer.setBrokerSet(brokerSet);
}
node.setConsumer(consumer);
// Create Test tuple collector
CollectorModule<String> collector =
dag.addOperator("TestMessageCollector", new CollectorModule<String>());
// Connect ports
dag.addStream("Kafka message", node.outputPort, collector.inputPort)
.setLocality(Locality.CONTAINER_LOCAL);
// Create local cluster
final LocalMode.Controller lc = lma.getController();
lc.setHeartbeatMonitoringEnabled(false);
lc.runAsync();
// Wait 30s for consumer finish consuming all the messages
Assert.assertTrue("TIMEOUT: 30s ", latch.await(30000, TimeUnit.MILLISECONDS));
// Check results
Assert.assertEquals("Collections size", 1, collections.size());
Assert.assertEquals("Tuple count", totalCount, collections.get(collector.inputPort.id).size());
logger.debug(
String.format(
"Number of emitted tuples: %d", collections.get(collector.inputPort.id).size()));
p.close();
lc.shutdown();
}
