public static void main(String[] args) {
if (!parseArgs(args)) throw new IllegalArgumentException("Parse arguments failed.");
SparkConf conf = new SparkConf().setAppName("Logistic Regression with SGD");
SparkContext sc = new SparkContext(conf);
JavaRDD<String> data = sc.textFile(inputFile, 1).toJavaRDD();
JavaRDD<LabeledPoint> training =
data.map(
new Function<String, LabeledPoint>() {
public LabeledPoint call(String line) {
String[] splits = line.split(",");
double[] features = new double[3];
try {
features[0] = Double.valueOf(splits[1]);
features[1] = Double.valueOf(splits[2]);
features[2] = Double.valueOf(splits[3]);
return new LabeledPoint(Double.valueOf(splits[3]), Vectors.dense(features));
} catch (NumberFormatException e) {
return null; // Nothing to do..
}
}
})
.filter(
new Function<LabeledPoint, Boolean>() {
public Boolean call(LabeledPoint p) {
return p != null;
}
})
.cache();
LogisticRegressionModel model = lrs.run(training.rdd());
model.save(sc, outputFile);
sc.stop();
}
