private void doSeekRetriableTopicPartitions() {
final Set<TopicPartition> retriableTopicPartitions = retryService.retriableTopicPartitions();
for (TopicPartition rtp : retriableTopicPartitions) {
final OffsetAndMetadata offsetAndMeta = acked.get(rtp).findNextCommitOffset();
if (offsetAndMeta != null) {
kafkaConsumer.seek(
rtp,
offsetAndMeta.offset()
+ 1); // seek to the next offset that is ready to commit in next commit cycle
} else {
kafkaConsumer.seek(
rtp, acked.get(rtp).committedOffset + 1); // Seek to last committed offset
}
}
}
/**
* Determines the offset of the next fetch. For failed batches lastBatchMeta is not null and
* contains the fetch offset of the failed batch. In this scenario the next fetch will take place
* at the offset of the failed batch. When the previous batch is successful, lastBatchMeta is
* null, and the offset of the next fetch is either the offset of the last commit to kafka, or if
* no commit was yet made, the offset dictated by {@link KafkaSpoutConfig.FirstPollOffsetStrategy}
*
* @return the offset of the next fetch
*/
private long seek(TopicPartition tp, KafkaTridentSpoutBatchMetadata<K, V> lastBatchMeta) {
if (lastBatchMeta != null) {
kafkaConsumer.seek(
tp,
lastBatchMeta.getLastOffset()
+ 1); // seek next offset after last offset from previous batch
LOG.debug("Seeking fetch offset to next offset after last offset from previous batch");
} else {
LOG.debug("Seeking fetch offset from firstPollOffsetStrategy and last commit to Kafka");
final OffsetAndMetadata committedOffset = kafkaConsumer.committed(tp);
if (committedOffset != null) { // offset was committed for this TopicPartition
if (firstPollOffsetStrategy.equals(EARLIEST)) {
kafkaConsumer.seekToBeginning(toArrayList(tp));
} else if (firstPollOffsetStrategy.equals(LATEST)) {
kafkaConsumer.seekToEnd(toArrayList(tp));
} else {
// By default polling starts at the last committed offset. +1 to point fetch to the first
// uncommitted offset.
kafkaConsumer.seek(tp, committedOffset.offset() + 1);
}
} else { // no commits have ever been done, so start at the beginning or end depending on the
// strategy
if (firstPollOffsetStrategy.equals(EARLIEST)
|| firstPollOffsetStrategy.equals(UNCOMMITTED_EARLIEST)) {
kafkaConsumer.seekToBeginning(toArrayList(tp));
} else if (firstPollOffsetStrategy.equals(LATEST)
|| firstPollOffsetStrategy.equals(UNCOMMITTED_LATEST)) {
kafkaConsumer.seekToEnd(toArrayList(tp));
}
}
}
final long fetchOffset = kafkaConsumer.position(tp);
LOG.debug("Set [fetchOffset = {}]", fetchOffset);
return fetchOffset;
}
