private void setupFlowExecution() {
int projectId = flow.getProjectId();
int version = flow.getVersion();
String flowId = flow.getFlowId();
// Add a bunch of common azkaban properties
commonProps = PropsUtils.addCommonFlowProperties(flow);
// Create execution dir
createLogger(flowId);
if (this.watcher != null) {
this.watcher.setLogger(logger);
}
logger.info(
"Running execid:"
+ execId
+ " flow:"
+ flowId
+ " project:"
+ projectId
+ " version:"
+ version);
if (pipelineExecId != null) {
logger.info(
"Running simulateously with " + pipelineExecId + ". Pipelining level " + pipelineLevel);
}
// The current thread is used for interrupting blocks
flowRunnerThread = Thread.currentThread();
flowRunnerThread.setName("FlowRunner-exec-" + flow.getExecutionId());
}
@SuppressWarnings("unchecked")
private void setupFlowExecution() {
int projectId = flow.getProjectId();
int version = flow.getVersion();
String flowId = flow.getFlowId();
// Add a bunch of common azkaban properties
Props commonFlowProps = PropsUtils.addCommonFlowProperties(null, flow);
if (flow.getJobSource() != null) {
String source = flow.getJobSource();
Props flowProps = sharedProps.get(source);
flowProps.setParent(commonFlowProps);
commonFlowProps = flowProps;
}
// If there are flow overrides, we apply them now.
Map<String, String> flowParam = flow.getExecutionOptions().getFlowParameters();
if (flowParam != null && !flowParam.isEmpty()) {
commonFlowProps = new Props(commonFlowProps, flowParam);
}
flow.setInputProps(commonFlowProps);
// Create execution dir
createLogger(flowId);
if (this.watcher != null) {
this.watcher.setLogger(logger);
}
logger.info(
"Running execid:"
+ execId
+ " flow:"
+ flowId
+ " project:"
+ projectId
+ " version:"
+ version);
if (pipelineExecId != null) {
logger.info(
"Running simulateously with " + pipelineExecId + ". Pipelining level " + pipelineLevel);
}
// The current thread is used for interrupting blocks
flowRunnerThread = Thread.currentThread();
flowRunnerThread.setName("FlowRunner-exec-" + flow.getExecutionId());
}
public void retryJobs(List<String> jobIds, String user) {
synchronized (mainSyncObj) {
for (String jobId : jobIds) {
ExecutableNode node = flow.getExecutableNode(jobId);
if (node == null) {
logger.error(
"Job "
+ jobId
+ " doesn't exist in execution "
+ flow.getExecutionId()
+ ". Cannot retry.");
continue;
}
if (Status.isStatusFinished(node.getStatus())) {
// Resets the status and increments the attempt number
node.resetForRetry();
reEnableDependents(node);
logger.info("Re-enabling job " + node.getJobId() + " attempt " + node.getAttempt());
} else {
logger.error("Cannot retry job " + jobId + " since it hasn't run yet. User " + user);
continue;
}
}
boolean isFailureFound = false;
for (ExecutableNode node : flow.getExecutableNodes()) {
Status nodeStatus = node.getStatus();
if (nodeStatus == Status.FAILED || nodeStatus == Status.KILLED) {
isFailureFound = true;
break;
}
}
if (!isFailureFound) {
flow.setStatus(Status.RUNNING);
flow.setUpdateTime(System.currentTimeMillis());
flowFailed = false;
}
updateFlow();
interrupt();
}
}
public FlowRunner(
ExecutableFlow flow,
ExecutorLoader executorLoader,
ProjectLoader projectLoader,
JobTypeManager jobtypeManager)
throws ExecutorManagerException {
this.execId = flow.getExecutionId();
this.flow = flow;
this.executorLoader = executorLoader;
this.projectLoader = projectLoader;
this.execDir = new File(flow.getExecutionPath());
this.jobtypeManager = jobtypeManager;
ExecutionOptions options = flow.getExecutionOptions();
this.pipelineLevel = options.getPipelineLevel();
this.pipelineExecId = options.getPipelineExecutionId();
this.proxyUsers = flow.getProxyUsers();
}
/**
* Constructor. If executorService is null, then it will create it's own for thread pools.
*
* @param flow
* @param executorLoader
* @param projectLoader
* @param jobtypeManager
* @param executorService
* @throws ExecutorManagerException
*/
public FlowRunner(
ExecutableFlow flow,
ExecutorLoader executorLoader,
ProjectLoader projectLoader,
JobTypeManager jobtypeManager,
ExecutorService executorService)
throws ExecutorManagerException {
this.execId = flow.getExecutionId();
this.flow = flow;
this.executorLoader = executorLoader;
this.projectLoader = projectLoader;
this.execDir = new File(flow.getExecutionPath());
this.jobtypeManager = jobtypeManager;
ExecutionOptions options = flow.getExecutionOptions();
this.pipelineLevel = options.getPipelineLevel();
this.pipelineExecId = options.getPipelineExecutionId();
this.failureAction = options.getFailureAction();
this.proxyUsers = flow.getProxyUsers();
this.executorService = executorService;
this.finishedNodes = new SwapQueue<ExecutableNode>();
}
