public static void main(String[] args) {
if (args.length < 4) {
System.err.println("Usage: JavaKafkaWordCount <zkQuorum> <group> <topics> <numThreads>");
System.exit(1);
}
StreamingExamples.setStreamingLogLevels();
// SparkConf sparkConf = new SparkConf().setAppName("JavaKafkaWordCount");
// sparkConf.setMaster("spark://60f81dc6426c:7077");
// SparkConf sparkConf = new
// SparkConf().setAppName("JavaKafkaWordCount").setMaster("spark://60f81dc6426c:7077");
// Create the context with a 1 second batch size
JavaStreamingContext jssc =
new JavaStreamingContext("local[4]", "JavaKafkaWordCount", new Duration(2000));
int numThreads = Integer.parseInt(args[3]);
Logger.getLogger("org").setLevel(Level.OFF);
Logger.getLogger("akka").setLevel(Level.OFF);
Map<String, Integer> topicMap = new HashMap<String, Integer>();
String[] topics = args[2].split(",");
for (String topic : topics) {
topicMap.put(topic, numThreads);
}
/* for(String t: topic)
{
topicMap.put(t, new Integer(3));
}*/
// NotSerializable notSerializable = new NotSerializable();
// JavaRDD<String> rdd = sc.textFile("/tmp/myfile");
// rdd.map(s -> notSerializable.doSomething(s)).collect();
JavaPairReceiverInputDStream<String, String> messages =
KafkaUtils.createStream(jssc, args[0], args[1], topicMap);
// JavaPairReceiverInputDStream<String, String> kafkaStream =
// KafkaUtils.createStream(jssc, "localhost:2181","streamingContext",
// topicMap);
System.out.println("Connection !!!!");
/*JavaDStream<String> data = messages.map(new Function<Tuple2<String, String>, String>()
{
public String call(Tuple2<String, String> message)
{
return message._2();
}
}
);*/
JavaDStream<String> lines =
messages.map(
new Function<Tuple2<String, String>, String>() {
@Override
public String call(Tuple2<String, String> tuple2) {
return tuple2._2();
}
});
JavaDStream<String> words =
lines.flatMap(
new FlatMapFunction<String, String>() {
@Override
public Iterable<String> call(String x) {
return Lists.newArrayList(SPACE.split(x));
}
});
JavaPairDStream<String, Integer> wordCounts =
words
.mapToPair(
new PairFunction<String, String, Integer>() {
@Override
public Tuple2<String, Integer> call(String s) {
return new Tuple2<String, Integer>(s, 1);
}
})
.reduceByKey(
new Function2<Integer, Integer, Integer>() {
@Override
public Integer call(Integer i1, Integer i2) {
return i1 + i2;
}
});
wordCounts.print();
jssc.start();
jssc.awaitTermination();
}