private long getFinalizedTimestampMillis(String topic) throws Exception {
final int numPartitions = mKafkaClient.getNumPartitions(topic);
long minTimestamp = Long.MAX_VALUE;
for (int partition = 0; partition < numPartitions; ++partition) {
TopicPartition topicPartition = new TopicPartition(topic, partition);
long timestamp = getFinalizedTimestampMillis(topicPartition);
LOG.info(
"finalized timestamp for topic "
+ topic
+ " partition "
+ partition
+ " is "
+ timestamp);
if (timestamp == -1) {
return -1;
} else {
if (timestamp < minTimestamp) {
minTimestamp = timestamp;
}
}
}
if (minTimestamp == Long.MAX_VALUE) {
return -1;
}
return minTimestamp;
}
private long getCommittedTimestampMillis(TopicPartition topicPartition) throws Exception {
Message message = mKafkaClient.getCommittedMessage(topicPartition);
if (message == null) {
LOG.error(
"No message found for topic "
+ topicPartition.getTopic()
+ " partition "
+ topicPartition.getPartition());
return -1;
}
return mMessageParser.extractTimestampMillis(message);
}
private long getLastTimestampMillis(TopicPartition topicPartition) throws Exception {
Message message = mKafkaClient.getLastMessage(topicPartition);
if (message == null) {
// This will happen if no messages have been posted to the given topic partition.
LOG.error(
"No message found for topic "
+ topicPartition.getTopic()
+ " partition "
+ topicPartition.getPartition());
return -1;
}
return mMessageParser.extractTimestampMillis(message);
}
private long getLastTimestampMillis(String topic) throws Exception {
final int numPartitions = mKafkaClient.getNumPartitions(topic);
long max_timestamp = Long.MIN_VALUE;
for (int partition = 0; partition < numPartitions; ++partition) {
TopicPartition topicPartition = new TopicPartition(topic, partition);
long timestamp = getLastTimestampMillis(topicPartition);
if (timestamp > max_timestamp) {
max_timestamp = timestamp;
}
}
if (max_timestamp == Long.MIN_VALUE) {
return -1;
}
return max_timestamp;
}
@Override
public void shutdown() throws Exception {
if (client != null) client.shutdown();
if (simpleClient != null) simpleClient.shutdown();
OperatorContext context = getOperatorContext();
trace.log(
TraceLevel.ALL,
"Operator "
+ context.getName()
+ " shutting down in PE: "
+ context.getPE().getPEId()
+ " in Job: "
+ context.getPE().getJobId());
// Must call super.shutdown()
super.shutdown();
}