private void submit(final FlowScript flow) {
LOG.debug("Submitting jobflow \"{}\": {}", flow.getId(), batchId);
FlowScriptTask task =
new FlowScriptTask(
flow,
doneQueue,
new Callable<Void>() {
@Override
public Void call() throws InterruptedException, IOException {
if (Thread.interrupted()) {
throw new InterruptedException();
}
String executionId = UUID.randomUUID().toString();
LOG.debug("Generated execution ID for \"{}\": {}", flow.getId(), executionId);
lock.beginFlow(flow.getId(), executionId);
executeFlow(batchId, flow, executionId);
lock.endFlow(flow.getId(), executionId);
LOG.debug("Completing jobflow \"{}\": {}", flow.getId(), batchId);
return null;
}
});
YSLOG.info("I01003", batchId, flow.getId());
executor.execute(task);
running.put(flow.getId(), task);
}
private void waitForComplete() throws InterruptedException, IOException {
LOG.debug("Waiting for running jobflows complete: {}", batchId);
FlowScriptTask done = doneQueue.take();
assert done.isDone();
FlowScript flow = done.script;
try {
done.get();
boolean blocked = blocking.remove(flow.getId());
assert blocked;
} catch (CancellationException e) {
YSLOG.info(e, "I01005", batchId, flow.getId());
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException) {
throw (IOException) e.getCause();
} else if (e.getCause() instanceof InterruptedException) {
throw (InterruptedException) e.getCause();
} else if (e.getCause() instanceof Error) {
throw (Error) e.getCause();
} else {
throw new IOException("Flow execution failed by unknown error", e);
}
} finally {
FlowScriptTask ran = running.remove(flow.getId());
assert ran != null;
}
}
void executeFlow(String batchId, FlowScript flow, String executionId)
throws InterruptedException, IOException {
assert batchId != null;
assert flow != null;
assert executionId != null;
YSLOG.info("I02000", batchId, flow.getId(), executionId);
long start = System.currentTimeMillis();
try {
if (skipFlows.contains(flow.getId())) {
YSLOG.info("I02002", batchId, flow.getId(), executionId);
return;
}
executePhase(batchId, flow, executionId, ExecutionPhase.SETUP);
boolean succeed = false;
try {
executePhase(batchId, flow, executionId, ExecutionPhase.INITIALIZE);
executePhase(batchId, flow, executionId, ExecutionPhase.IMPORT);
executePhase(batchId, flow, executionId, ExecutionPhase.PROLOGUE);
executePhase(batchId, flow, executionId, ExecutionPhase.MAIN);
executePhase(batchId, flow, executionId, ExecutionPhase.EPILOGUE);
executePhase(batchId, flow, executionId, ExecutionPhase.EXPORT);
succeed = true;
} finally {
if (succeed) {
executePhase(batchId, flow, executionId, ExecutionPhase.FINALIZE);
} else {
YSLOG.info("I02003", batchId, flow.getId(), executionId);
try {
executePhase(batchId, flow, executionId, ExecutionPhase.FINALIZE);
} catch (Exception e) {
YSLOG.warn(e, "W02002", batchId, flow.getId(), executionId);
}
}
}
try {
executePhase(batchId, flow, executionId, ExecutionPhase.CLEANUP);
} catch (Exception e) {
YSLOG.warn(e, "W02003", batchId, flow.getId(), executionId);
}
YSLOG.info("I02001", batchId, flow.getId(), executionId);
} catch (IOException e) {
YSLOG.error(e, "E02001", batchId, flow.getId(), executionId);
throw e;
} catch (InterruptedException e) {
YSLOG.warn(e, "W02001", batchId, flow.getId(), executionId);
throw e;
} finally {
long end = System.currentTimeMillis();
YSLOG.info("I02999", batchId, flow.getId(), executionId, end - start);
}
}
private void executePhase(
String batchId, FlowScript flow, String executionId, ExecutionPhase phase)
throws InterruptedException, IOException {
ExecutionContext context =
new ExecutionContext(
batchId,
flow.getId(),
executionId,
phase,
batchArguments,
subprocessEnvironmentVaritables);
Set<ExecutionScript> scripts = flow.getScripts().get(phase);
assert scripts != null;
executePhase(context, scripts);
}
BatchScheduler(
String batchId, BatchScript batchScript, ExecutionLock lock, ExecutorService executor) {
assert batchId != null;
assert batchScript != null;
assert lock != null;
assert executor != null;
this.batchId = batchId;
this.flows = new LinkedList<FlowScript>(batchScript.getAllFlows());
this.lock = lock;
this.executor = executor;
this.running = new HashMap<String, FlowScriptTask>();
this.blocking = new HashSet<String>();
for (FlowScript flow : flows) {
blocking.add(flow.getId());
}
this.doneQueue = new LinkedBlockingQueue<FlowScriptTask>();
}
private boolean submit() {
LOG.debug("Submitting waiting jobflows: {}", batchId);
boolean submitted = false;
for (Iterator<FlowScript> iter = flows.iterator(); iter.hasNext(); ) {
FlowScript flow = iter.next();
boolean blocked = false;
for (String blockerId : flow.getBlockerIds()) {
if (blocking.contains(blockerId)) {
blocked = true;
break;
}
}
if (blocked == false) {
submit(flow);
iter.remove();
submitted = true;
}
}
return submitted;
}
/**
* Executes a target phase.
*
* @param context the current context
* @throws InterruptedException if interrupted during this execution
* @throws IOException if failed to execute target phase
* @throws IllegalArgumentException if some parameters were {@code null}
* @since 0.2.5
*/
public void executePhase(ExecutionContext context) throws InterruptedException, IOException {
if (context == null) {
throw new IllegalArgumentException("context must not be null"); // $NON-NLS-1$
}
FlowScript flow = script.findFlow(context.getFlowId());
if (flow == null) {
throw new IllegalArgumentException(
MessageFormat.format(
"Flow is undefined: batchId={0}, flowId={1}, executionId={2}",
context.getBatchId(), context.getFlowId(), context.getExecutionId()));
}
Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase());
ExecutionLock lock = acquireExecutionLock(context.getBatchId());
try {
lock.beginFlow(context.getFlowId(), context.getExecutionId());
executePhase(context, executions);
lock.endFlow(context.getFlowId(), context.getExecutionId());
} finally {
lock.close();
}
}