public void kill(String user) {
synchronized (mainSyncObj) {
logger.info("Flow killed by " + user);
flow.setStatus(Status.KILLED);
kill();
updateFlow();
}
interrupt();
}
public void resume(String user) {
synchronized (mainSyncObj) {
if (!flowPaused) {
logger.info("Cannot resume flow that isn't paused");
} else {
logger.info("Flow resumed by " + user);
flowPaused = false;
if (flowFailed) {
flow.setStatus(Status.FAILED_FINISHING);
} else if (flowCancelled) {
flow.setStatus(Status.KILLED);
} else {
flow.setStatus(Status.RUNNING);
}
updateFlow();
}
}
}
public void pause(String user) {
synchronized (mainSyncObj) {
if (!flowFinished) {
logger.info("Flow paused by " + user);
flowPaused = true;
flow.setStatus(Status.PAUSED);
updateFlow();
} else {
logger.info("Cannot pause finished flow. Called by user " + user);
}
}
interrupt();
}
public void retryJobs(List<String> jobIds, String user) {
synchronized (mainSyncObj) {
for (String jobId : jobIds) {
ExecutableNode node = flow.getExecutableNode(jobId);
if (node == null) {
logger.error(
"Job "
+ jobId
+ " doesn't exist in execution "
+ flow.getExecutionId()
+ ". Cannot retry.");
continue;
}
if (Status.isStatusFinished(node.getStatus())) {
// Resets the status and increments the attempt number
node.resetForRetry();
reEnableDependents(node);
logger.info("Re-enabling job " + node.getJobId() + " attempt " + node.getAttempt());
} else {
logger.error("Cannot retry job " + jobId + " since it hasn't run yet. User " + user);
continue;
}
}
boolean isFailureFound = false;
for (ExecutableNode node : flow.getExecutableNodes()) {
Status nodeStatus = node.getStatus();
if (nodeStatus == Status.FAILED || nodeStatus == Status.KILLED) {
isFailureFound = true;
break;
}
}
if (!isFailureFound) {
flow.setStatus(Status.RUNNING);
flow.setUpdateTime(System.currentTimeMillis());
flowFailed = false;
}
updateFlow();
interrupt();
}
}
private void cancel() {
synchronized (mainSyncObj) {
logger.info("Cancel has been called on flow " + execId);
flowPaused = false;
flowCancelled = true;
if (watcher != null) {
logger.info("Watcher is attached. Stopping watcher.");
watcher.stopWatcher();
logger.info("Watcher cancelled status is " + watcher.isWatchCancelled());
}
logger.info("Cancelling " + activeJobRunners.size() + " jobs.");
for (JobRunner runner : activeJobRunners.values()) {
runner.cancel();
}
if (flow.getStatus() != Status.FAILED && flow.getStatus() != Status.FAILED_FINISHING) {
logger.info("Setting flow status to " + Status.KILLED.toString());
flow.setStatus(Status.KILLED);
}
}
}
public void run() {
try {
if (this.executorService == null) {
this.executorService = Executors.newFixedThreadPool(numJobThreads);
}
setupFlowExecution();
flow.setStartTime(System.currentTimeMillis());
updateFlowReference();
logger.info("Updating initial flow directory.");
updateFlow();
logger.info("Fetching job and shared properties.");
loadAllProperties();
this.fireEventListeners(Event.create(this, Type.FLOW_STARTED));
runFlow();
} catch (Throwable t) {
if (logger != null) {
logger.error("An error has occurred during the running of the flow. Quiting.", t);
}
flow.setStatus(Status.FAILED);
} finally {
if (watcher != null) {
logger.info("Watcher is attached. Stopping watcher.");
watcher.stopWatcher();
logger.info("Watcher cancelled status is " + watcher.isWatchCancelled());
}
flow.setEndTime(System.currentTimeMillis());
logger.info("Setting end time for flow " + execId + " to " + System.currentTimeMillis());
closeLogger();
updateFlow();
this.fireEventListeners(Event.create(this, Type.FLOW_FINISHED));
}
}
/**
* Main method that executes the jobs.
*
* @throws Exception
*/
private void runFlow() throws Exception {
logger.info("Starting flows");
flow.setStatus(Status.RUNNING);
updateFlow();
while (!flowFinished) {
synchronized (mainSyncObj) {
if (flowPaused) {
try {
mainSyncObj.wait(CHECK_WAIT_MS);
} catch (InterruptedException e) {
}
continue;
} else {
List<ExecutableNode> jobsReadyToRun = findReadyJobsToRun();
if (!jobsReadyToRun.isEmpty() && !flowCancelled) {
for (ExecutableNode node : jobsReadyToRun) {
long currentTime = System.currentTimeMillis();
// Queue a job only if it's ready to run.
if (node.getStatus() == Status.READY) {
// Collect output props from the job's dependencies.
Props outputProps = collectOutputProps(node);
node.setStatus(Status.QUEUED);
JobRunner runner = createJobRunner(node, outputProps);
logger.info("Submitting job " + node.getJobId() + " to run.");
try {
executorService.submit(runner);
jobRunners.put(node.getJobId(), runner);
activeJobRunners.put(node.getJobId(), runner);
} catch (RejectedExecutionException e) {
logger.error(e);
}
;
} // If killed, then auto complete and KILL
else if (node.getStatus() == Status.KILLED) {
logger.info("Killing " + node.getJobId() + " due to prior errors.");
node.setStartTime(currentTime);
node.setEndTime(currentTime);
fireEventListeners(Event.create(this, Type.JOB_FINISHED, node));
} // If disabled, then we auto skip
else if (node.getStatus() == Status.DISABLED) {
logger.info("Skipping disabled job " + node.getJobId() + ".");
node.setStartTime(currentTime);
node.setEndTime(currentTime);
node.setStatus(Status.SKIPPED);
fireEventListeners(Event.create(this, Type.JOB_FINISHED, node));
}
}
updateFlow();
} else {
if (isFlowFinished() || flowCancelled) {
flowFinished = true;
break;
}
try {
mainSyncObj.wait(CHECK_WAIT_MS);
} catch (InterruptedException e) {
}
}
}
}
}
if (flowCancelled) {
try {
logger.info("Flow was force cancelled cleaning up.");
for (JobRunner activeRunner : activeJobRunners.values()) {
activeRunner.cancel();
}
for (ExecutableNode node : flow.getExecutableNodes()) {
if (Status.isStatusFinished(node.getStatus())) {
continue;
} else if (node.getStatus() == Status.DISABLED) {
node.setStatus(Status.SKIPPED);
} else {
node.setStatus(Status.KILLED);
}
fireEventListeners(Event.create(this, Type.JOB_FINISHED, node));
}
} catch (Exception e) {
logger.error(e);
}
updateFlow();
}
logger.info("Finishing up flow. Awaiting Termination");
executorService.shutdown();
synchronized (mainSyncObj) {
switch (flow.getStatus()) {
case FAILED_FINISHING:
logger.info("Setting flow status to Failed.");
flow.setStatus(Status.FAILED);
case FAILED:
case KILLED:
logger.info("Flow is set to " + flow.getStatus().toString());
break;
default:
flow.setStatus(Status.SUCCEEDED);
logger.info("Flow is set to " + flow.getStatus().toString());
}
}
}
