// Emits one tuple per record
// @return true if tuple was emitted
private boolean emitTupleIfNotEmitted(ConsumerRecord<K, V> record) {
final TopicPartition tp = new TopicPartition(record.topic(), record.partition());
final KafkaSpoutMessageId msgId = new KafkaSpoutMessageId(record);
if (acked.containsKey(tp) && acked.get(tp).contains(msgId)) { // has been acked
LOG.trace("Tuple for record [{}] has already been acked. Skipping", record);
} else if (emitted.contains(msgId)) { // has been emitted and it's pending ack or fail
LOG.trace("Tuple for record [{}] has already been emitted. Skipping", record);
} else {
boolean isScheduled = retryService.isScheduled(msgId);
if (!isScheduled
|| retryService.isReady(
msgId)) { // not scheduled <=> never failed (i.e. never emitted) or ready to be
// retried
final List<Object> tuple = tuplesBuilder.buildTuple(record);
kafkaSpoutStreams.emit(collector, tuple, msgId);
emitted.add(msgId);
numUncommittedOffsets++;
if (isScheduled) { // Was scheduled for retry, now being re-emitted. Remove from schedule.
retryService.remove(msgId);
}
LOG.trace("Emitted tuple [{}] for record [{}]", tuple, record);
return true;
}
}
return false;
}
private void doSeekRetriableTopicPartitions() {
final Set<TopicPartition> retriableTopicPartitions = retryService.retriableTopicPartitions();
for (TopicPartition rtp : retriableTopicPartitions) {
final OffsetAndMetadata offsetAndMeta = acked.get(rtp).findNextCommitOffset();
if (offsetAndMeta != null) {
kafkaConsumer.seek(
rtp,
offsetAndMeta.offset()
+ 1); // seek to the next offset that is ready to commit in next commit cycle
} else {
kafkaConsumer.seek(
rtp, acked.get(rtp).committedOffset + 1); // Seek to last committed offset
}
}
}
@Override
public void fail(Object messageId) {
final KafkaSpoutMessageId msgId = (KafkaSpoutMessageId) messageId;
if (!emitted.contains(msgId)) {
LOG.debug(
"Received fail for tuple this spout is no longer tracking. Partitions may have been reassigned. Ignoring message [{}]",
msgId);
return;
}
if (msgId.numFails() < maxRetries) {
emitted.remove(msgId);
msgId.incrementNumFails();
retryService.schedule(msgId);
} else { // limit to max number of retries
LOG.debug("Reached maximum number of retries. Message [{}] being marked as acked.", msgId);
ack(msgId);
}
}
