/**
* Executes a target flow. If execution is failed except on {@link ExecutionPhase#SETUP setup},
* {@link ExecutionPhase#FINALIZE finalize}, or {@link ExecutionPhase#CLEANUP cleanup}, then the
* {@code finalize} phase will be executed for recovery before execution is aborted.
*
* @param batchId target batch ID
* @param flowId target flow ID
* @param executionId target execution ID
* @throws InterruptedException if interrupted during this execution
* @throws IOException if failed to execute target flow
* @throws IllegalArgumentException if some parameters were {@code null}
*/
public void executeFlow(String batchId, String flowId, String executionId)
throws InterruptedException, IOException {
if (batchId == null) {
throw new IllegalArgumentException("batchId must not be null"); // $NON-NLS-1$
}
if (flowId == null) {
throw new IllegalArgumentException("flowId must not be null"); // $NON-NLS-1$
}
if (executionId == null) {
throw new IllegalArgumentException("executionId must not be null"); // $NON-NLS-1$
}
FlowScript flow = script.findFlow(flowId);
if (flow == null) {
throw new IllegalArgumentException(
MessageFormat.format(
"Flow is undefined: batchId={0}, flowId={1}, executionId={2}",
batchId, flowId, executionId));
}
ExecutionLock lock = acquireExecutionLock(batchId);
try {
lock.beginFlow(flowId, executionId);
executeFlow(batchId, flow, executionId);
lock.endFlow(flowId, executionId);
} finally {
lock.close();
}
}
/**
* Loads profile and create a new {@link ExecutionTask}.
*
* @param profile target profile
* @param script target script
* @param batchArguments current batch arguments
* @param yaessArguments current controll arguments
* @return the created task
* @throws InterruptedException if interrupted while configuring services
* @throws IOException if failed to configure services
* @throws IllegalArgumentException if some parameters were {@code null}
* @since 0.2.6
*/
public static ExecutionTask load(
YaessProfile profile,
Properties script,
Map<String, String> batchArguments,
Map<String, String> yaessArguments)
throws InterruptedException, IOException {
if (profile == null) {
throw new IllegalArgumentException("profile must not be null"); // $NON-NLS-1$
}
if (script == null) {
throw new IllegalArgumentException("script must not be null"); // $NON-NLS-1$
}
if (batchArguments == null) {
throw new IllegalArgumentException("batchArguments must not be null"); // $NON-NLS-1$
}
if (yaessArguments == null) {
throw new IllegalArgumentException("yaessArguments must not be null"); // $NON-NLS-1$
}
LOG.debug("Loading execution monitor feature");
ExecutionMonitorProvider monitors = profile.getMonitors().newInstance();
LOG.debug("Loading execution lock feature");
ExecutionLockProvider locks = profile.getLocks().newInstance();
LOG.debug("Loading job scheduling feature");
JobScheduler scheduler = profile.getScheduler().newInstance();
LOG.debug("Loading hadoop execution feature");
HadoopScriptHandler hadoopHandler = profile.getHadoopHandler().newInstance();
LOG.debug("Loading command execution features");
Map<String, CommandScriptHandler> commandHandlers = new HashMap<String, CommandScriptHandler>();
for (Map.Entry<String, ServiceProfile<CommandScriptHandler>> entry :
profile.getCommandHandlers().entrySet()) {
commandHandlers.put(entry.getKey(), entry.getValue().newInstance());
}
LOG.debug("Extracting batch script");
BatchScript batch = BatchScript.load(script);
ExecutionTask result =
new ExecutionTask(
monitors, locks, scheduler, hadoopHandler, commandHandlers, batch, batchArguments);
LOG.debug("Applying definitions");
Map<String, String> copyDefinitions = new TreeMap<String, String>(yaessArguments);
consumeRuntimeContext(result, copyDefinitions, batch);
consumeSkipFlows(result, copyDefinitions, batch);
consumeSerializeFlows(result, copyDefinitions, batch);
checkRest(copyDefinitions);
return result;
}
private static void consumeRuntimeContext(
ExecutionTask result, Map<String, String> copyDefinitions, BatchScript script) {
assert result != null;
assert copyDefinitions != null;
assert script != null;
RuntimeContext rc = RuntimeContext.get().batchId(script.getId()).buildId(script.getBuildId());
Ternary dryRunResult = consumeBoolean(copyDefinitions, KEY_VERIFY_DRYRUN);
if (dryRunResult == Ternary.TRUE) {
rc = rc.mode(ExecutionMode.SIMULATION);
} else if (dryRunResult == Ternary.FALSE) {
rc = rc.mode(ExecutionMode.PRODUCTION);
}
Ternary verify = consumeBoolean(copyDefinitions, KEY_VERIFY_APPLICATION);
if (verify == Ternary.FALSE) {
rc = rc.buildId(null);
}
result.runtimeContext = rc;
result.getEnv().putAll(rc.unapply());
}
BatchScheduler(
String batchId, BatchScript batchScript, ExecutionLock lock, ExecutorService executor) {
assert batchId != null;
assert batchScript != null;
assert lock != null;
assert executor != null;
this.batchId = batchId;
this.flows = new LinkedList<FlowScript>(batchScript.getAllFlows());
this.lock = lock;
this.executor = executor;
this.running = new HashMap<String, FlowScriptTask>();
this.blocking = new HashSet<String>();
for (FlowScript flow : flows) {
blocking.add(flow.getId());
}
this.doneQueue = new LinkedBlockingQueue<FlowScriptTask>();
}
/**
* Executes a target phase.
*
* @param context the current context
* @throws InterruptedException if interrupted during this execution
* @throws IOException if failed to execute target phase
* @throws IllegalArgumentException if some parameters were {@code null}
* @since 0.2.5
*/
public void executePhase(ExecutionContext context) throws InterruptedException, IOException {
if (context == null) {
throw new IllegalArgumentException("context must not be null"); // $NON-NLS-1$
}
FlowScript flow = script.findFlow(context.getFlowId());
if (flow == null) {
throw new IllegalArgumentException(
MessageFormat.format(
"Flow is undefined: batchId={0}, flowId={1}, executionId={2}",
context.getBatchId(), context.getFlowId(), context.getExecutionId()));
}
Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase());
ExecutionLock lock = acquireExecutionLock(context.getBatchId());
try {
lock.beginFlow(context.getFlowId(), context.getExecutionId());
executePhase(context, executions);
lock.endFlow(context.getFlowId(), context.getExecutionId());
} finally {
lock.close();
}
}
private static void consumeSkipFlows(
ExecutionTask task, Map<String, String> copyDefinitions, BatchScript script) {
assert task != null;
assert copyDefinitions != null;
assert script != null;
String flows = copyDefinitions.remove(KEY_SKIP_FLOWS);
if (flows == null || flows.trim().isEmpty()) {
return;
}
LOG.debug("Definition: {}={}", KEY_SKIP_FLOWS, flows);
for (String flowIdCandidate : flows.split(",")) {
String flowId = flowIdCandidate.trim();
if (flowId.isEmpty() == false) {
FlowScript flow = script.findFlow(flowId);
if (flow == null) {
throw new IllegalArgumentException(
MessageFormat.format(
"Unknown flowId in definition {0} : {1}", KEY_SKIP_FLOWS, flowId));
}
task.skipFlows.add(flowId);
}
}
}