/** * Start fetching messages for stream from kafkaConsumer with given fetch rate etc. NOT thread * safe!!!!! * * @throws cn.edu.sdu.cs.starry.kafkaConsumer.exception.ConsumerLogException */ public void startFetchingAndPushing(boolean uptToDate, int fetchSize, long fetchRate) throws KafkaCommunicationException { if (uptToDate) { setAllOffsetsTo(kafka.api.OffsetRequest.LatestTime()); } int fetchTimes = 0; while (!shutdown) { try { Thread.sleep(fetchRate); } catch (InterruptedException e) { e.printStackTrace(); } List<KafkaMessage> messageAndOffsetList = fetchMessage(fetchSize); for (KafkaMessage message : messageAndOffsetList) { try { messageSender.sendMessage(message); } catch (Exception ex) { LOG.error("Sending failed! [{}]", ex.getMessage(), ex); continue; } } if (messageAndOffsetList.size() > 0) { fetchTimes = (fetchTimes + 1) % logFlushInterval; } if (fetchTimes == 0 && messageAndOffsetList.size() > 0) { try { fetchOperator.flushOffsets(); } catch (ConsumerLogException e) { LOG.warn("Flush offsets error!"); LOG.warn(e.getMessage()); fetchOperator.handleLogError(); } } messageAndOffsetList.clear(); } }
@Override public void close() { shutdown = true; messageSender.close(); }