コード例 #1
0
ファイル: SparkRuntime.java プロジェクト: arthurv89/crunch
 public SparkRuntime(
     SparkPipeline pipeline,
     JavaSparkContext sparkContext,
     Configuration conf,
     Map<PCollectionImpl<?>, Set<Target>> outputTargets,
     Map<PCollectionImpl<?>, MaterializableIterable> toMaterialize,
     Map<PCollection<?>, StorageLevel> toCache,
     Map<PipelineCallable<?>, Set<Target>> allPipelineCallables) {
   this.pipeline = pipeline;
   this.sparkContext = sparkContext;
   this.conf = conf;
   this.counters =
       sparkContext.accumulator(
           Maps.<String, Map<String, Long>>newHashMap(), new CounterAccumulatorParam());
   this.ctxt =
       new SparkRuntimeContext(
           sparkContext.appName(),
           counters,
           sparkContext.broadcast(WritableUtils.toByteArray(conf)));
   this.outputTargets = Maps.newTreeMap(DEPTH_COMPARATOR);
   this.outputTargets.putAll(outputTargets);
   this.toMaterialize = toMaterialize;
   this.toCache = toCache;
   this.allPipelineCallables = allPipelineCallables;
   this.activePipelineCallables = allPipelineCallables.keySet();
   this.status.set(Status.READY);
   this.monitorThread =
       new Thread(
           new Runnable() {
             @Override
             public void run() {
               monitorLoop();
             }
           });
 }
コード例 #2
0
  public static JavaRDD<String> getJavaRDD(JavaSparkContext sparkContext) {

    System.out.println("Converting" + sparkContext.version() + sparkContext.appName());

    JavaRDD<String> testJRDD = null;
    try {
      testJRDD =
          sparkContext.textFile(
              "/Users/shawnkyzer/Documents/aleph2_analytic_services_R/hs_err_pid2930.log");
    } catch (Exception e) {
      System.out.println(e.fillInStackTrace());
    }
    System.out.println("Converting");
    return testJRDD;
  }