private void run() {
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
// Hard coding single threaded consumer
topicCountMap.put(inputTopic, 1);
Properties props = createConsumerConfig(zookeeper, groupId, url);
VerifiableProperties vProps = new VerifiableProperties(props);
// Create decoders for key and value
KafkaAvroDecoder avroDecoder = new KafkaAvroDecoder(vProps);
StringDecoder stringDecoder = new StringDecoder(new VerifiableProperties());
KafkaStream stream =
consumer
.createMessageStreams(topicCountMap, stringDecoder, avroDecoder)
.get(inputTopic)
.get(0);
ConsumerIterator it = stream.iterator();
System.out.println("Ready to start iterating wih properties: " + props.toString());
System.out.println("Reading topic:" + inputTopic);
while (it.hasNext()) {
MessageAndMetadata messageAndMetadata = it.next();
String ip = (String) messageAndMetadata.key();
// Once we release a new version of the avro deserializer that can return SpecificData, the
// deep copy will be unnecessary
GenericRecord genericEvent = (GenericRecord) messageAndMetadata.message();
LogLine event = (LogLine) SpecificData.get().deepCopy(LogLine.SCHEMA$, genericEvent);
SessionState oldState = state.get(ip);
int sessionId = 0;
if (oldState == null) {
state.put(ip, new SessionState(event.getTimestamp(), 0));
} else {
sessionId = oldState.getSessionId();
// if the old timestamp is more than 30 minutes older than new one, we have a new session
if (oldState.getLastConnection() < event.getTimestamp() - sessionLengthMs)
sessionId = sessionId + 1;
SessionState newState = new SessionState(event.getTimestamp(), sessionId);
state.put(ip, newState);
}
event.setSessionid(sessionId);
System.out.println(event.toString());
ProducerRecord<String, LogLine> record =
new ProducerRecord<String, LogLine>(outputTopic, event.getIp().toString(), event);
producer.send(record);
}
}
