public static void main(String[] args) {
// Create a Spark Context.
SparkConf conf = new SparkConf().setAppName("Activity").set("spark.eventLog.enabled", "true");
;
JavaSparkContext sc = new JavaSparkContext(conf);
JavaStreamingContext jssc = new JavaStreamingContext(sc, STREAM_INTERVAL);
String TOPIC = "activityevent";
String zkQuorum = "localhost:2181";
String group = "1";
Map<String, Integer> topicMap = new HashMap<String, Integer>();
topicMap.put(TOPIC, 1);
JavaPairReceiverInputDStream<String, String> messages =
KafkaUtils.createStream(jssc, zkQuorum, group, topicMap);
// messages.print();
JavaDStream<String> activitydatastream =
messages.map(
new Function<Tuple2<String, String>, String>() {
@Override
public String call(Tuple2<String, String> tuple2) {
return tuple2._2();
}
});
final Long teamWindowDurationMs = Durations.minutes(1).milliseconds();
JavaDStream<Activity> ActivityEntryDStream = activitydatastream.map(Activity::parseFromLine);
JavaPairDStream<WithTimestamp<String>, Double> ActivityWindowDStream =
ActivityEntryDStream.mapToPair(
windows ->
new Tuple2<>(
WithTimestamp.create(
windows.getActivity(),
// Apply Fixed Window by rounding the timestamp down to the nearest
// multiple of the window size
(convertMillsecs(windows.getTimestamp()) / teamWindowDurationMs)
* teamWindowDurationMs),
windows.getXaxis()))
.reduceByKey(SUM_REDUCER);
ActivityWindowDStream.print();
jssc.start();
jssc.awaitTermination();
// jssc.close();
sc.stop();
sc.close();
}
