private Props loadJobProps(ExecutableNode node) throws IOException {
Props props = null;
String source = node.getJobSource();
if (source == null) {
return null;
}
// load the override props if any
try {
props =
projectLoader.fetchProjectProperty(
flow.getProjectId(), flow.getVersion(), node.getId() + ".jor");
} catch (ProjectManagerException e) {
e.printStackTrace();
logger.error("Error loading job override property for job " + node.getId());
}
File path = new File(execDir, source);
if (props == null) {
// if no override prop, load the original one on disk
try {
props = new Props(null, path);
} catch (IOException e) {
e.printStackTrace();
logger.error("Error loading job file " + source + " for job " + node.getId());
}
}
// setting this fake source as this will be used to determine the location of log files.
if (path.getPath() != null) {
props.setSource(path.getPath());
}
return props;
}
private void setupFlowExecution() {
int projectId = flow.getProjectId();
int version = flow.getVersion();
String flowId = flow.getFlowId();
// Add a bunch of common azkaban properties
commonProps = PropsUtils.addCommonFlowProperties(flow);
// Create execution dir
createLogger(flowId);
if (this.watcher != null) {
this.watcher.setLogger(logger);
}
logger.info(
"Running execid:"
+ execId
+ " flow:"
+ flowId
+ " project:"
+ projectId
+ " version:"
+ version);
if (pipelineExecId != null) {
logger.info(
"Running simulateously with " + pipelineExecId + ". Pipelining level " + pipelineLevel);
}
// The current thread is used for interrupting blocks
flowRunnerThread = Thread.currentThread();
flowRunnerThread.setName("FlowRunner-exec-" + flow.getExecutionId());
}
@SuppressWarnings("unchecked")
private void setupFlowExecution() {
int projectId = flow.getProjectId();
int version = flow.getVersion();
String flowId = flow.getFlowId();
// Add a bunch of common azkaban properties
Props commonFlowProps = PropsUtils.addCommonFlowProperties(null, flow);
if (flow.getJobSource() != null) {
String source = flow.getJobSource();
Props flowProps = sharedProps.get(source);
flowProps.setParent(commonFlowProps);
commonFlowProps = flowProps;
}
// If there are flow overrides, we apply them now.
Map<String, String> flowParam = flow.getExecutionOptions().getFlowParameters();
if (flowParam != null && !flowParam.isEmpty()) {
commonFlowProps = new Props(commonFlowProps, flowParam);
}
flow.setInputProps(commonFlowProps);
// Create execution dir
createLogger(flowId);
if (this.watcher != null) {
this.watcher.setLogger(logger);
}
logger.info(
"Running execid:"
+ execId
+ " flow:"
+ flowId
+ " project:"
+ projectId
+ " version:"
+ version);
if (pipelineExecId != null) {
logger.info(
"Running simulateously with " + pipelineExecId + ". Pipelining level " + pipelineLevel);
}
// The current thread is used for interrupting blocks
flowRunnerThread = Thread.currentThread();
flowRunnerThread.setName("FlowRunner-exec-" + flow.getExecutionId());
}
private JobRunner createJobRunner(ExecutableNode node, Props previousOutput) {
String source = node.getJobPropsSource();
String propsSource = node.getPropsSource();
// If no properties are set, we just set the global properties.
Props parentProps = propsSource == null ? globalProps : sharedProps.get(propsSource);
// Set up overrides
ExecutionOptions options = flow.getExecutionOptions();
@SuppressWarnings("unchecked")
Props flowProps = new Props(null, options.getFlowParameters());
flowProps.putAll(commonProps);
flowProps.setParent(parentProps);
parentProps = flowProps;
// We add the previous job output and put into this props.
if (previousOutput != null) {
Props earliestParent = previousOutput.getEarliestAncestor();
earliestParent.setParent(parentProps);
parentProps = previousOutput;
}
// Load job file.
File path = new File(execDir, source);
Props prop = null;
// load the override props if any
try {
prop =
projectLoader.fetchProjectProperty(
flow.getProjectId(), flow.getVersion(), node.getJobId() + ".jor");
} catch (ProjectManagerException e) {
e.printStackTrace();
logger.error("Error loading job override property for job " + node.getJobId());
}
if (prop == null) {
// if no override prop, load the original one on disk
try {
prop = new Props(null, path);
} catch (IOException e) {
e.printStackTrace();
logger.error("Error loading job file " + source + " for job " + node.getJobId());
}
}
// setting this fake source as this will be used to determine the location of log files.
prop.setSource(path.getPath());
prop.setParent(parentProps);
JobRunner jobRunner =
new JobRunner(node, prop, path.getParentFile(), executorLoader, jobtypeManager);
if (watcher != null) {
jobRunner.setPipeline(watcher, pipelineLevel);
}
if (validateUserProxy) {
jobRunner.setValidatedProxyUsers(proxyUsers);
}
jobRunner.setDelayStart(node.getDelayedExecution());
jobRunner.setLogSettings(logger, jobLogFileSize, jobLogNumFiles);
jobRunner.addListener(listener);
return jobRunner;
}