private static void execute() throws InterruptedException {
KafkaConsumer<String, String> consumer = createConsumer();
// Subscribe to all partition in that topic. 'assign' could be used here
// instead of 'subscribe' to subscribe to specific partition.
consumer.subscribe(Arrays.asList("normal-topic"));
processRecords(consumer);
}
public static void main(String[] args) throws UnknownHostException {
Properties props = new Properties();
props.put("bootstrap.servers", "kafka01:9092,kafka02:9092,kafka03:9092");
props.put("group.id", "test");
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
props.put("session.timeout.ms", "30000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props);
consumer.subscribe(Arrays.asList("apt-receive1"));
List<TopicPartition> partitions = new ArrayList<>();
// partitions.add(new TopicPartition("apt-receive1", 2));
// partitions.add(new TopicPartition("apt-receive1", 13));
// consumer.assign(partitions);
for (int i = 0; i < 10000; i++) {
ConsumerRecords<String, byte[]> records = consumer.poll(100);
System.out.println(i + ": " + records.count());
for (ConsumerRecord<String, byte[]> record : records) {
// System.out.println(record.key());
bloom[Integer.parseInt(record.key())] = 1;
}
// if (sum == 10000) {
// System.out.println("sum=" + sum);
// break;
// }
}
for (int j = 0; j < 10_000_000; j++) {
if (bloom[j] == 0) {
System.err.println("" + j);
}
}
consumer.close();
System.err.println("Finish!");
}
/** @see stream.io.AbstractStream#init() */
@Override
public void init() throws Exception {
super.init();
if (group == null) {
group = UUID.randomUUID().toString();
}
Properties props = new Properties();
props.put("zookeeper.connect", zookeeper);
props.put("metadata.broker.list", broker);
props.put("bootstrap.servers", broker);
props.put("group.id", group);
props.put("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "10000");
log.debug("Creating kafka consumer...");
final KafkaConsumer<byte[], byte[]> kc = new KafkaConsumer<byte[], byte[]>(props);
consumer = kc;
log.info("Subscribing to topic '{}'", topic);
consumer.subscribe(Arrays.asList(topic));
log.info("Using codec {}", valueCodec);
List<PartitionInfo> partitionInfo = consumer.partitionsFor(topic);
log.debug("topic '{}' has {} partitions", topic, partitionInfo.size());
pollThread =
new Thread() {
public void run() {
try {
running.set(true);
while (running.get()) {
ConsumerRecords<byte[], byte[]> messages = consumer.poll(100L);
log.debug("Polled new messages: {}", messages);
while (messages == null || messages.count() < 1) {
messages = consumer.poll(1000L);
log.debug("Polled new messages: {}", messages);
}
if (messages != null) {
Iterator<ConsumerRecord<byte[], byte[]>> it = messages.iterator();
while (it.hasNext()) {
ConsumerRecord<byte[], byte[]> record = it.next();
Object key = record.key();
byte[] data = record.value();
Data item = null;
if (valueCodec != null) {
log.debug("de-serializing item using {}", valueCodec);
item = valueCodec.decode(data);
} else {
item = DataFactory.create();
if (key != null) {
item.put("@kafka:key", (Serializable) key);
}
item.put("@kafka:value", data);
}
item.put("@kafka:topic", record.topic());
item.put("@kafka:partition", record.partition());
queue.add(item);
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
log.info("Closing topic-stream reader...");
queue.add(endOfStream);
}
}
};
pollThread.setDaemon(true);
pollThread.start();
Signals.register(
new Hook() {
@Override
public void signal(int flags) {
log.info("Received shutdown-signal!");
running.set(false);
pollThread.interrupt();
log.info("Adding EOF to queue...");
queue.add(endOfStream);
}
});
}
@Override
public void drive() {
// A Consumer is not thread-safe
// {@see
// http://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html}
// {@see
// http://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html#multithreaded}
try (KafkaConsumer<byte[], byte[]> consumer =
new KafkaConsumer<>(consumerDefinition.getKafkaConfig())) {
String topic = consumerDefinition.getTopic();
log.info("Subscribing to {}", topic);
if (consumerRebalanceListener == null) {
consumer.subscribe(Collections.singletonList(topic));
} else {
consumer.subscribe(Collections.singletonList(topic), consumerRebalanceListener);
}
long messagesToReceive = consumerDefinition.getMessagesToReceive();
log.info("Expecting {} messages", messagesToReceive);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
do {
ConsumerRecords<byte[], byte[]> records =
consumer.poll(consumerDefinition.getPollTimeout());
if (records == null) {
throw new IllegalStateException("null ConsumerRecords polled");
} else {
if (records.count() == 0) {
try {
log.info("No records fetched, pausing");
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
} else {
if (log.isTraceEnabled()) {
log.trace("Fetched {} records", records.count());
}
for (ConsumerRecord<byte[], byte[]> record : records) {
recordsFetched += 1;
applyReceiveDelay();
if (recordsFetched % consumerDefinition.getReportReceivedEvery() == 0) {
log.info("Received {} messages", recordsFetched);
}
}
}
}
if (isShutdownRequested()) {
break;
}
stopWatch.split();
} while ((recordsFetched < messagesToReceive)
&& (stopWatch.getSplitTime() < consumerDefinition.getTestRunTimeout()));
stopWatch.stop();
if (isShutdownRequested()) {
log.info("Shutting down");
} else {
long runTime = stopWatch.getTime();
log.info("Done. Consumer received {} msgs in {} ms", messagesToReceive, runTime);
double averageThroughput = (1000d / runTime) * messagesToReceive;
log.info("Average throughput: {} msg/s", averageThroughput);
}
} finally {
log.debug("Consumer closed");
if (completionLatch != null) {
completionLatch.countDown();
}
}
}
