Пример #1
0
 public void produce(SimplifiedLog message, String topicName) {
   if (producer == null)
     producer =
         KafkaUtils.createProducer(
             KafkaUtils.createZkClient(zkServer.getConnectString()), ASYNC, false);
   producer.send(new KeyedMessage<>(topicName, message.getHostName(), message));
   LOGGER.debug("Sent message: {}", message);
 }
Пример #2
0
  public void start() throws Exception {
    zkServer = new TestingServer(zkPort, true);
    zkClient = KafkaUtils.createZkClient(zkServer.getConnectString());

    kafkaServer = new KafkaServerStartable(new KafkaConfig(getServerProperties()));
    kafkaServer.startup();
    LOGGER.debug("Started Kafka server at port {}", kafkaPort);
  }
Пример #3
0
  public List<SimplifiedLog> consume(String topicName, int topicPartitions, int expectedMsg)
      throws InterruptedException {
    ConsumerConnector consumer =
        KafkaUtils.createConsumer(zkServer.getConnectString(), "test_group", "1");
    List<KafkaStream<String, SimplifiedLog>> streams =
        KafkaUtils.getConsumerStreams(consumer, topicName, topicPartitions);

    List<Callable<List<SimplifiedLog>>> tasks = new ArrayList<>();
    streams.forEach(stream -> tasks.add(createConsumerThread(stream.iterator(), expectedMsg)));

    ExecutorService executor = Executors.newFixedThreadPool(streams.size());
    List<Future<List<SimplifiedLog>>> futures =
        executor.invokeAll(tasks, 5 * expectedMsg, TimeUnit.SECONDS);

    List<SimplifiedLog> received = getResultsFromFutures(futures);
    consumer.shutdown();
    return received;
  }