/**
* Start fetching messages for stream from kafkaConsumer with given fetch rate etc. NOT thread
* safe!!!!!
*
* @throws cn.edu.sdu.cs.starry.kafkaConsumer.exception.ConsumerLogException
*/
public void startFetchingAndPushing(boolean uptToDate, int fetchSize, long fetchRate)
throws KafkaCommunicationException {
if (uptToDate) {
setAllOffsetsTo(kafka.api.OffsetRequest.LatestTime());
}
int fetchTimes = 0;
while (!shutdown) {
try {
Thread.sleep(fetchRate);
} catch (InterruptedException e) {
e.printStackTrace();
}
List<KafkaMessage> messageAndOffsetList = fetchMessage(fetchSize);
for (KafkaMessage message : messageAndOffsetList) {
try {
messageSender.sendMessage(message);
} catch (Exception ex) {
LOG.error("Sending failed! [{}]", ex.getMessage(), ex);
continue;
}
}
if (messageAndOffsetList.size() > 0) {
fetchTimes = (fetchTimes + 1) % logFlushInterval;
}
if (fetchTimes == 0 && messageAndOffsetList.size() > 0) {
try {
fetchOperator.flushOffsets();
} catch (ConsumerLogException e) {
LOG.warn("Flush offsets error!");
LOG.warn(e.getMessage());
fetchOperator.handleLogError();
}
}
messageAndOffsetList.clear();
}
}
@Override
public void close() {
shutdown = true;
messageSender.close();
}