@Override
public KafkaTridentSpoutBatchMetadata<K, V> emitPartitionBatch(
TransactionAttempt tx,
TridentCollector collector,
KafkaTridentSpoutTopicPartition partitionTs,
KafkaTridentSpoutBatchMetadata<K, V> lastBatch) {
LOG.debug(
"Emitting batch: [transaction = {}], [partition = {}], [collector = {}], [lastBatchMetadata = {}]",
tx,
partitionTs,
collector,
lastBatch);
final TopicPartition topicPartition = partitionTs.getTopicPartition();
KafkaTridentSpoutBatchMetadata<K, V> currentBatch = lastBatch;
Collection<TopicPartition> pausedTopicPartitions = Collections.emptySet();
try {
// pause other topic partitions to only poll from current topic partition
pausedTopicPartitions = pauseTopicPartitions(topicPartition);
seek(topicPartition, lastBatch);
// poll
final ConsumerRecords<K, V> records = kafkaConsumer.poll(pollTimeoutMs);
LOG.debug("Polled [{}] records from Kafka.", records.count());
if (!records.isEmpty()) {
emitTuples(collector, records);
// build new metadata
currentBatch = new KafkaTridentSpoutBatchMetadata<>(topicPartition, records, lastBatch);
}
} finally {
kafkaConsumer.resume(pausedTopicPartitions);
LOG.trace("Resumed topic partitions [{}]", pausedTopicPartitions);
}
LOG.debug("Current batch metadata {}", currentBatch);
return currentBatch;
}
private static void processRecords(KafkaConsumer<String, String> consumer)
throws InterruptedException {
while (true) {
ConsumerRecords<String, String> records = consumer.poll(100);
long lastOffset = 0;
for (ConsumerRecord<String, String> record : records) {
System.out.printf(
"\n\roffset = %d, key = %s, value = %s", record.offset(), record.key(), record.value());
lastOffset = record.offset();
}
System.out.println("lastOffset read: " + lastOffset);
process();
// Below call is important to control the offset commit. Do this call after you
// finish processing the business process to get the at least once guarantee.
consumer.commitSync();
}
}
public static void main(String[] args) throws UnknownHostException {
Properties props = new Properties();
props.put("bootstrap.servers", "kafka01:9092,kafka02:9092,kafka03:9092");
props.put("group.id", "test");
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
props.put("session.timeout.ms", "30000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props);
consumer.subscribe(Arrays.asList("apt-receive1"));
List<TopicPartition> partitions = new ArrayList<>();
// partitions.add(new TopicPartition("apt-receive1", 2));
// partitions.add(new TopicPartition("apt-receive1", 13));
// consumer.assign(partitions);
for (int i = 0; i < 10000; i++) {
ConsumerRecords<String, byte[]> records = consumer.poll(100);
System.out.println(i + ": " + records.count());
for (ConsumerRecord<String, byte[]> record : records) {
// System.out.println(record.key());
bloom[Integer.parseInt(record.key())] = 1;
}
// if (sum == 10000) {
// System.out.println("sum=" + sum);
// break;
// }
}
for (int j = 0; j < 10_000_000; j++) {
if (bloom[j] == 0) {
System.err.println("" + j);
}
}
consumer.close();
System.err.println("Finish!");
}
@Override
public void drive() {
// A Consumer is not thread-safe
// {@see
// http://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html}
// {@see
// http://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html#multithreaded}
try (KafkaConsumer<byte[], byte[]> consumer =
new KafkaConsumer<>(consumerDefinition.getKafkaConfig())) {
String topic = consumerDefinition.getTopic();
log.info("Subscribing to {}", topic);
if (consumerRebalanceListener == null) {
consumer.subscribe(Collections.singletonList(topic));
} else {
consumer.subscribe(Collections.singletonList(topic), consumerRebalanceListener);
}
long messagesToReceive = consumerDefinition.getMessagesToReceive();
log.info("Expecting {} messages", messagesToReceive);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
do {
ConsumerRecords<byte[], byte[]> records =
consumer.poll(consumerDefinition.getPollTimeout());
if (records == null) {
throw new IllegalStateException("null ConsumerRecords polled");
} else {
if (records.count() == 0) {
try {
log.info("No records fetched, pausing");
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
} else {
if (log.isTraceEnabled()) {
log.trace("Fetched {} records", records.count());
}
for (ConsumerRecord<byte[], byte[]> record : records) {
recordsFetched += 1;
applyReceiveDelay();
if (recordsFetched % consumerDefinition.getReportReceivedEvery() == 0) {
log.info("Received {} messages", recordsFetched);
}
}
}
}
if (isShutdownRequested()) {
break;
}
stopWatch.split();
} while ((recordsFetched < messagesToReceive)
&& (stopWatch.getSplitTime() < consumerDefinition.getTestRunTimeout()));
stopWatch.stop();
if (isShutdownRequested()) {
log.info("Shutting down");
} else {
long runTime = stopWatch.getTime();
log.info("Done. Consumer received {} msgs in {} ms", messagesToReceive, runTime);
double averageThroughput = (1000d / runTime) * messagesToReceive;
log.info("Average throughput: {} msg/s", averageThroughput);
}
} finally {
log.debug("Consumer closed");
if (completionLatch != null) {
completionLatch.countDown();
}
}
}