Exemplo n.º 1
0
 /**
  * Start fetching messages for stream from kafkaConsumer with given fetch rate etc. NOT thread
  * safe!!!!!
  *
  * @throws cn.edu.sdu.cs.starry.kafkaConsumer.exception.ConsumerLogException
  */
 public void startFetchingAndPushing(boolean uptToDate, int fetchSize, long fetchRate)
     throws KafkaCommunicationException {
   if (uptToDate) {
     setAllOffsetsTo(kafka.api.OffsetRequest.LatestTime());
   }
   int fetchTimes = 0;
   while (!shutdown) {
     try {
       Thread.sleep(fetchRate);
     } catch (InterruptedException e) {
       e.printStackTrace();
     }
     List<KafkaMessage> messageAndOffsetList = fetchMessage(fetchSize);
     for (KafkaMessage message : messageAndOffsetList) {
       try {
         messageSender.sendMessage(message);
       } catch (Exception ex) {
         LOG.error("Sending failed! [{}]", ex.getMessage(), ex);
         continue;
       }
     }
     if (messageAndOffsetList.size() > 0) {
       fetchTimes = (fetchTimes + 1) % logFlushInterval;
     }
     if (fetchTimes == 0 && messageAndOffsetList.size() > 0) {
       try {
         fetchOperator.flushOffsets();
       } catch (ConsumerLogException e) {
         LOG.warn("Flush offsets error!");
         LOG.warn(e.getMessage());
         fetchOperator.handleLogError();
       }
     }
     messageAndOffsetList.clear();
   }
 }
Exemplo n.º 2
0
 @Override
 public void close() {
   shutdown = true;
   messageSender.close();
 }