/**
* Executes a target flow. If execution is failed except on {@link ExecutionPhase#SETUP setup},
* {@link ExecutionPhase#FINALIZE finalize}, or {@link ExecutionPhase#CLEANUP cleanup}, then the
* {@code finalize} phase will be executed for recovery before execution is aborted.
*
* @param batchId target batch ID
* @param flowId target flow ID
* @param executionId target execution ID
* @throws InterruptedException if interrupted during this execution
* @throws IOException if failed to execute target flow
* @throws IllegalArgumentException if some parameters were {@code null}
*/
public void executeFlow(String batchId, String flowId, String executionId)
throws InterruptedException, IOException {
if (batchId == null) {
throw new IllegalArgumentException("batchId must not be null"); // $NON-NLS-1$
}
if (flowId == null) {
throw new IllegalArgumentException("flowId must not be null"); // $NON-NLS-1$
}
if (executionId == null) {
throw new IllegalArgumentException("executionId must not be null"); // $NON-NLS-1$
}
FlowScript flow = script.findFlow(flowId);
if (flow == null) {
throw new IllegalArgumentException(
MessageFormat.format(
"Flow is undefined: batchId={0}, flowId={1}, executionId={2}",
batchId, flowId, executionId));
}
ExecutionLock lock = acquireExecutionLock(batchId);
try {
lock.beginFlow(flowId, executionId);
executeFlow(batchId, flow, executionId);
lock.endFlow(flowId, executionId);
} finally {
lock.close();
}
}
/**
* Executes a target flow. If execution is failed except on {@link ExecutionPhase#SETUP setup},
* {@link ExecutionPhase#FINALIZE finalize}, or {@link ExecutionPhase#CLEANUP cleanup}, then the
* {@code finalize} phase will be executed for recovery before the target flow execution is
* aborted. Execution ID for each flow will automatically generated.
*
* @param batchId target batch ID
* @throws InterruptedException if interrupted during this execution
* @throws IOException if failed to execute target batch
* @throws IllegalArgumentException if some parameters were {@code null}
*/
public void executeBatch(String batchId) throws InterruptedException, IOException {
if (batchId == null) {
throw new IllegalArgumentException("batchId must not be null"); // $NON-NLS-1$
}
ExecutorService executor = createJobflowExecutor(batchId);
YSLOG.info("I01000", batchId);
long start = System.currentTimeMillis();
try {
ExecutionLock lock = acquireExecutionLock(batchId);
try {
BatchScheduler batchScheduler = new BatchScheduler(batchId, script, lock, executor);
batchScheduler.run();
} finally {
lock.close();
}
YSLOG.info("I01001", batchId);
} catch (IOException e) {
YSLOG.error(e, "E01001", batchId);
throw e;
} catch (InterruptedException e) {
YSLOG.warn(e, "W01001", batchId);
throw e;
} finally {
long end = System.currentTimeMillis();
YSLOG.info("I01999", batchId, end - start);
}
}
/**
* Lock provider for batch.
*
* @throws Exception if failed
*/
@Test
public void batch() throws Exception {
Map<String, String> conf = new HashMap<>();
conf.put(BasicLockProvider.KEY_DIRECTORY, folder.getRoot().getAbsolutePath());
conf.put(ExecutionLockProvider.KEY_SCOPE, ExecutionLock.Scope.BATCH.getSymbol());
ServiceProfile<ExecutionLockProvider> profile =
new ServiceProfile<>(
"testing",
BasicLockProvider.class,
conf,
ProfileContext.system(getClass().getClassLoader()));
ExecutionLockProvider instance1 = profile.newInstance();
ExecutionLockProvider instance2 = profile.newInstance();
try (ExecutionLock lock = instance1.newInstance("batch1")) {
lock.beginFlow("flow1", "exec1");
try {
instance2.newInstance("batch1");
fail("cannot run same batch");
} catch (IOException e) {
// ok.
}
try (ExecutionLock other = instance2.newInstance("batch2")) {
// can acquire any flow/exec lock
other.beginFlow("flow2", "exec1");
other.endFlow("flow2", "exec1");
other.beginFlow("flow1", "exec2");
other.endFlow("flow1", "exec2");
other.beginFlow("flow2", "exec2");
other.endFlow("flow2", "exec2");
}
}
}
/**
* Missing directory config.
*
* @throws Exception if failed
*/
@Test
public void missing_directory() throws Exception {
File lockDir = folder.newFolder("missing");
Assume.assumeThat(lockDir.delete(), is(true));
Map<String, String> conf = new HashMap<>();
conf.put(BasicLockProvider.KEY_DIRECTORY, lockDir.getAbsolutePath());
ServiceProfile<ExecutionLockProvider> profile =
new ServiceProfile<>(
"testing",
BasicLockProvider.class,
conf,
ProfileContext.system(getClass().getClassLoader()));
ExecutionLockProvider instance = profile.newInstance();
try (ExecutionLock lock = instance.newInstance("batch")) {
lock.beginFlow("a", "b");
lock.endFlow("a", "b");
}
}
/**
* Executes a target phase.
*
* @param context the current context
* @throws InterruptedException if interrupted during this execution
* @throws IOException if failed to execute target phase
* @throws IllegalArgumentException if some parameters were {@code null}
* @since 0.2.5
*/
public void executePhase(ExecutionContext context) throws InterruptedException, IOException {
if (context == null) {
throw new IllegalArgumentException("context must not be null"); // $NON-NLS-1$
}
FlowScript flow = script.findFlow(context.getFlowId());
if (flow == null) {
throw new IllegalArgumentException(
MessageFormat.format(
"Flow is undefined: batchId={0}, flowId={1}, executionId={2}",
context.getBatchId(), context.getFlowId(), context.getExecutionId()));
}
Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase());
ExecutionLock lock = acquireExecutionLock(context.getBatchId());
try {
lock.beginFlow(context.getFlowId(), context.getExecutionId());
executePhase(context, executions);
lock.endFlow(context.getFlowId(), context.getExecutionId());
} finally {
lock.close();
}
}
/**
* Simple testing.
*
* @throws Exception if failed
*/
@Test
public void simple() throws Exception {
File lockDir = folder.getRoot();
int start = lockDir.list().length;
Map<String, String> conf = new HashMap<>();
conf.put(BasicLockProvider.KEY_DIRECTORY, lockDir.getAbsolutePath());
ServiceProfile<ExecutionLockProvider> profile =
new ServiceProfile<>(
"testing",
BasicLockProvider.class,
conf,
ProfileContext.system(getClass().getClassLoader()));
ExecutionLockProvider instance = profile.newInstance();
try (ExecutionLock lock = instance.newInstance("batch")) {
lock.beginFlow("flow", "exec");
assertThat(lockDir.list().length, is(greaterThan(start)));
}
assertThat(lockDir.list().length, is(start));
}
/**
* Configure using variable.
*
* @throws Exception if failed
*/
@Test
public void with_variable() throws Exception {
File lockDir = folder.getRoot();
int start = lockDir.list().length;
VariableResolver var =
new VariableResolver(Collections.singletonMap("LOCATION", lockDir.getAbsolutePath()));
Map<String, String> conf = new HashMap<>();
conf.put(BasicLockProvider.KEY_DIRECTORY, "${LOCATION}");
ServiceProfile<ExecutionLockProvider> profile =
new ServiceProfile<>(
"testing",
BasicLockProvider.class,
conf,
new ProfileContext(getClass().getClassLoader(), var));
ExecutionLockProvider instance = profile.newInstance();
try (ExecutionLock lock = instance.newInstance("batch")) {
lock.beginFlow("flow", "exec");
assertThat(lockDir.list().length, is(greaterThan(start)));
}
assertThat(lockDir.list().length, is(start));
}