public static void main(String[] args) {
JavaSparkContext sc = new JavaSparkContext();
Configuration conf = sc.hadoopConfiguration();
conf.set("fs.swift.impl", "org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem");
conf.set("fs.swift.service.test.auth.url", "http://163.17.136.246:5000/v2.0/tokens");
conf.set("fs.swift.service.test.auth.endpoint.prefix", "endpoints");
conf.set("fs.swift.service.test.http.port", "8080");
conf.set("fs.swift.service.test.region", "RegionOne");
conf.set("fs.swift.service.test.public", "true");
conf.set("fs.swift.service.test.tenant", "big-data");
conf.set("fs.swift.service.test.username", "k753357");
conf.set("fs.swift.service.test.password", "k753357");
JavaRDD<String> rawRDD = sc.textFile(args[0]);
rawRDD.saveAsTextFile("swift://testfile.test/file/");
}
public static void main(String[] args) {
SparkConf sparkconf =
new SparkConf()
.setAppName("Simple Application")
.setMaster("spark://1.245.77.10:7077")
.set(
"spark.driver.extraClassPath",
"E:/installprogram/spark-1.5.2-bin-hadoop2.4/libthirdparty/*")
.set(
"spark.executor.extraClassPath",
"E:/installprogram/spark-1.5.2-bin-hadoop2.4/libthirdparty/*")
.set("fs.default.name", "file:///");
JavaSparkContext sc = new JavaSparkContext(sparkconf);
Configuration hadoopConfig = sc.hadoopConfiguration();
hadoopConfig.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
hadoopConfig.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
// sc.addJar("e:/installprogram/spark-1.5.2-bin-hadoop2.4/libthirdparty/jmatrw-0.2.jar");
// sc.addJar("e:/installprogram/spark-1.5.2-bin-hadoop2.4/libthirdparty/jmatrw4spark-0.2.jar");
/*JavaRDD<Double> matrdd2 = sc.parallelize(Arrays.asList(1.0, 3.0, 2.0));
System.out.println("Start counting parallelize...");
long values = matrdd2.count();
System.out.println("Value count of parallelize is " + values);*/
JavaPairRDD<Long, Double> matrdd =
sc.newAPIHadoopFile(
"e:/tmp/vecRow03_x256.mat",
JMATFileInputFormat.class,
Long.class,
Double.class,
hadoopConfig);
System.out.println("Start job...");
long values = matrdd.count();
System.out.println("Value count of hadoop is " + values);
sc.stop();
sc.close();
}
