예제 #1
0
  @Override
  public ProgramController run(Program program, ProgramOptions options) {
    // Extract and verify parameters
    final ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");

    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(
        processorType == ProgramType.SPARK, "Only Spark process type is supported.");

    final SparkSpecification spec = appSpec.getSpark().get(program.getName());
    Preconditions.checkNotNull(spec, "Missing SparkSpecification for %s", program.getName());

    // Optionally get runId. If the spark started by other program (e.g. Workflow), it inherit the
    // runId.
    Arguments arguments = options.getArguments();
    RunId runId = RunIds.fromString(arguments.getOption(ProgramOptionConstants.RUN_ID));

    long logicalStartTime =
        arguments.hasOption(ProgramOptionConstants.LOGICAL_START_TIME)
            ? Long.parseLong(arguments.getOption(ProgramOptionConstants.LOGICAL_START_TIME))
            : System.currentTimeMillis();

    WorkflowToken workflowToken = null;
    if (arguments.hasOption(ProgramOptionConstants.WORKFLOW_TOKEN)) {
      workflowToken =
          GSON.fromJson(
              arguments.getOption(ProgramOptionConstants.WORKFLOW_TOKEN), BasicWorkflowToken.class);
    }

    ClientSparkContext context =
        new ClientSparkContext(
            program,
            runId,
            logicalStartTime,
            options.getUserArguments().asMap(),
            new TransactionContext(txSystemClient),
            datasetFramework,
            discoveryServiceClient,
            metricsCollectionService,
            workflowToken);

    Spark spark;
    try {
      spark =
          new InstantiatorFactory(false).get(TypeToken.of(program.<Spark>getMainClass())).create();

      // Fields injection
      Reflections.visit(
          spark,
          TypeToken.of(spark.getClass()),
          new PropertyFieldSetter(spec.getProperties()),
          new DataSetFieldSetter(context),
          new MetricsFieldSetter(context.getMetrics()));
    } catch (Exception e) {
      LOG.error("Failed to instantiate Spark class for {}", spec.getClassName(), e);
      throw Throwables.propagate(e);
    }

    Service sparkRuntimeService =
        new SparkRuntimeService(
            cConf,
            hConf,
            spark,
            new SparkContextFactory(hConf, context, datasetFramework, streamAdmin),
            program.getJarLocation(),
            txSystemClient);

    sparkRuntimeService.addListener(
        createRuntimeServiceListener(program.getId(), runId, arguments),
        Threads.SAME_THREAD_EXECUTOR);
    ProgramController controller = new SparkProgramController(sparkRuntimeService, context);

    LOG.info("Starting Spark Job: {}", context.toString());
    sparkRuntimeService.start();
    return controller;
  }
예제 #2
0
  /** Creates a service listener to reactor on state changes on {@link SparkRuntimeService}. */
  private Service.Listener createRuntimeServiceListener(
      final Id.Program programId, final RunId runId, Arguments arguments) {

    final String twillRunId = arguments.getOption(ProgramOptionConstants.TWILL_RUN_ID);
    final String workflowName = arguments.getOption(ProgramOptionConstants.WORKFLOW_NAME);
    final String workflowNodeId = arguments.getOption(ProgramOptionConstants.WORKFLOW_NODE_ID);
    final String workflowRunId = arguments.getOption(ProgramOptionConstants.WORKFLOW_RUN_ID);

    return new ServiceListenerAdapter() {
      @Override
      public void starting() {
        // Get start time from RunId
        long startTimeInSeconds = RunIds.getTime(runId, TimeUnit.SECONDS);
        if (startTimeInSeconds == -1) {
          // If RunId is not time-based, use current time as start time
          startTimeInSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
        }

        if (workflowName == null) {
          store.setStart(programId, runId.getId(), startTimeInSeconds, null, twillRunId);
        } else {
          // Program started by Workflow
          store.setWorkflowProgramStart(
              programId,
              runId.getId(),
              workflowName,
              workflowRunId,
              workflowNodeId,
              startTimeInSeconds,
              null,
              twillRunId);
        }
      }

      @Override
      public void terminated(Service.State from) {
        if (from == Service.State.STOPPING) {
          // Service was killed
          store.setStop(
              programId,
              runId.getId(),
              TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
              ProgramController.State.KILLED.getRunStatus());
        } else {
          // Service completed by itself.
          store.setStop(
              programId,
              runId.getId(),
              TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
              ProgramController.State.COMPLETED.getRunStatus());
        }
      }

      @Override
      public void failed(Service.State from, Throwable failure) {
        store.setStop(
            programId,
            runId.getId(),
            TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
            ProgramController.State.ERROR.getRunStatus());
      }
    };
  }