public SparkRuntime(
SparkPipeline pipeline,
JavaSparkContext sparkContext,
Configuration conf,
Map<PCollectionImpl<?>, Set<Target>> outputTargets,
Map<PCollectionImpl<?>, MaterializableIterable> toMaterialize,
Map<PCollection<?>, StorageLevel> toCache,
Map<PipelineCallable<?>, Set<Target>> allPipelineCallables) {
this.pipeline = pipeline;
this.sparkContext = sparkContext;
this.conf = conf;
this.counters =
sparkContext.accumulator(
Maps.<String, Map<String, Long>>newHashMap(), new CounterAccumulatorParam());
this.ctxt =
new SparkRuntimeContext(
sparkContext.appName(),
counters,
sparkContext.broadcast(WritableUtils.toByteArray(conf)));
this.outputTargets = Maps.newTreeMap(DEPTH_COMPARATOR);
this.outputTargets.putAll(outputTargets);
this.toMaterialize = toMaterialize;
this.toCache = toCache;
this.allPipelineCallables = allPipelineCallables;
this.activePipelineCallables = allPipelineCallables.keySet();
this.status.set(Status.READY);
this.monitorThread =
new Thread(
new Runnable() {
@Override
public void run() {
monitorLoop();
}
});
}
public static JavaRDD<String> getJavaRDD(JavaSparkContext sparkContext) {
System.out.println("Converting" + sparkContext.version() + sparkContext.appName());
JavaRDD<String> testJRDD = null;
try {
testJRDD =
sparkContext.textFile(
"/Users/shawnkyzer/Documents/aleph2_analytic_services_R/hs_err_pid2930.log");
} catch (Exception e) {
System.out.println(e.fillInStackTrace());
}
System.out.println("Converting");
return testJRDD;
}
