@Override
public ProgramController run(Program program, ProgramOptions options) {
// Extract and verify parameters
final ApplicationSpecification appSpec = program.getApplicationSpecification();
Preconditions.checkNotNull(appSpec, "Missing application specification.");
ProgramType processorType = program.getType();
Preconditions.checkNotNull(processorType, "Missing processor type.");
Preconditions.checkArgument(
processorType == ProgramType.SPARK, "Only Spark process type is supported.");
final SparkSpecification spec = appSpec.getSpark().get(program.getName());
Preconditions.checkNotNull(spec, "Missing SparkSpecification for %s", program.getName());
// Optionally get runId. If the spark started by other program (e.g. Workflow), it inherit the
// runId.
Arguments arguments = options.getArguments();
RunId runId = RunIds.fromString(arguments.getOption(ProgramOptionConstants.RUN_ID));
long logicalStartTime =
arguments.hasOption(ProgramOptionConstants.LOGICAL_START_TIME)
? Long.parseLong(arguments.getOption(ProgramOptionConstants.LOGICAL_START_TIME))
: System.currentTimeMillis();
WorkflowToken workflowToken = null;
if (arguments.hasOption(ProgramOptionConstants.WORKFLOW_TOKEN)) {
workflowToken =
GSON.fromJson(
arguments.getOption(ProgramOptionConstants.WORKFLOW_TOKEN), BasicWorkflowToken.class);
}
ClientSparkContext context =
new ClientSparkContext(
program,
runId,
logicalStartTime,
options.getUserArguments().asMap(),
new TransactionContext(txSystemClient),
datasetFramework,
discoveryServiceClient,
metricsCollectionService,
workflowToken);
Spark spark;
try {
spark =
new InstantiatorFactory(false).get(TypeToken.of(program.<Spark>getMainClass())).create();
// Fields injection
Reflections.visit(
spark,
TypeToken.of(spark.getClass()),
new PropertyFieldSetter(spec.getProperties()),
new DataSetFieldSetter(context),
new MetricsFieldSetter(context.getMetrics()));
} catch (Exception e) {
LOG.error("Failed to instantiate Spark class for {}", spec.getClassName(), e);
throw Throwables.propagate(e);
}
Service sparkRuntimeService =
new SparkRuntimeService(
cConf,
hConf,
spark,
new SparkContextFactory(hConf, context, datasetFramework, streamAdmin),
program.getJarLocation(),
txSystemClient);
sparkRuntimeService.addListener(
createRuntimeServiceListener(program.getId(), runId, arguments),
Threads.SAME_THREAD_EXECUTOR);
ProgramController controller = new SparkProgramController(sparkRuntimeService, context);
LOG.info("Starting Spark Job: {}", context.toString());
sparkRuntimeService.start();
return controller;
}
/** Creates a service listener to reactor on state changes on {@link SparkRuntimeService}. */
private Service.Listener createRuntimeServiceListener(
final Id.Program programId, final RunId runId, Arguments arguments) {
final String twillRunId = arguments.getOption(ProgramOptionConstants.TWILL_RUN_ID);
final String workflowName = arguments.getOption(ProgramOptionConstants.WORKFLOW_NAME);
final String workflowNodeId = arguments.getOption(ProgramOptionConstants.WORKFLOW_NODE_ID);
final String workflowRunId = arguments.getOption(ProgramOptionConstants.WORKFLOW_RUN_ID);
return new ServiceListenerAdapter() {
@Override
public void starting() {
// Get start time from RunId
long startTimeInSeconds = RunIds.getTime(runId, TimeUnit.SECONDS);
if (startTimeInSeconds == -1) {
// If RunId is not time-based, use current time as start time
startTimeInSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
}
if (workflowName == null) {
store.setStart(programId, runId.getId(), startTimeInSeconds, null, twillRunId);
} else {
// Program started by Workflow
store.setWorkflowProgramStart(
programId,
runId.getId(),
workflowName,
workflowRunId,
workflowNodeId,
startTimeInSeconds,
null,
twillRunId);
}
}
@Override
public void terminated(Service.State from) {
if (from == Service.State.STOPPING) {
// Service was killed
store.setStop(
programId,
runId.getId(),
TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
ProgramController.State.KILLED.getRunStatus());
} else {
// Service completed by itself.
store.setStop(
programId,
runId.getId(),
TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
ProgramController.State.COMPLETED.getRunStatus());
}
}
@Override
public void failed(Service.State from, Throwable failure) {
store.setStop(
programId,
runId.getId(),
TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
ProgramController.State.ERROR.getRunStatus());
}
};
}
