Exemple #1
0
  private Props loadJobProps(ExecutableNode node) throws IOException {
    Props props = null;
    String source = node.getJobSource();
    if (source == null) {
      return null;
    }

    // load the override props if any
    try {
      props =
          projectLoader.fetchProjectProperty(
              flow.getProjectId(), flow.getVersion(), node.getId() + ".jor");
    } catch (ProjectManagerException e) {
      e.printStackTrace();
      logger.error("Error loading job override property for job " + node.getId());
    }

    File path = new File(execDir, source);
    if (props == null) {
      // if no override prop, load the original one on disk
      try {
        props = new Props(null, path);
      } catch (IOException e) {
        e.printStackTrace();
        logger.error("Error loading job file " + source + " for job " + node.getId());
      }
    }
    // setting this fake source as this will be used to determine the location of log files.
    if (path.getPath() != null) {
      props.setSource(path.getPath());
    }
    return props;
  }
Exemple #2
0
  private void setupFlowExecution() {
    int projectId = flow.getProjectId();
    int version = flow.getVersion();
    String flowId = flow.getFlowId();

    // Add a bunch of common azkaban properties
    commonProps = PropsUtils.addCommonFlowProperties(flow);

    // Create execution dir
    createLogger(flowId);

    if (this.watcher != null) {
      this.watcher.setLogger(logger);
    }

    logger.info(
        "Running execid:"
            + execId
            + " flow:"
            + flowId
            + " project:"
            + projectId
            + " version:"
            + version);
    if (pipelineExecId != null) {
      logger.info(
          "Running simulateously with " + pipelineExecId + ". Pipelining level " + pipelineLevel);
    }

    // The current thread is used for interrupting blocks
    flowRunnerThread = Thread.currentThread();
    flowRunnerThread.setName("FlowRunner-exec-" + flow.getExecutionId());
  }
Exemple #3
0
  @SuppressWarnings("unchecked")
  private void setupFlowExecution() {
    int projectId = flow.getProjectId();
    int version = flow.getVersion();
    String flowId = flow.getFlowId();

    // Add a bunch of common azkaban properties
    Props commonFlowProps = PropsUtils.addCommonFlowProperties(null, flow);

    if (flow.getJobSource() != null) {
      String source = flow.getJobSource();
      Props flowProps = sharedProps.get(source);
      flowProps.setParent(commonFlowProps);
      commonFlowProps = flowProps;
    }

    // If there are flow overrides, we apply them now.
    Map<String, String> flowParam = flow.getExecutionOptions().getFlowParameters();
    if (flowParam != null && !flowParam.isEmpty()) {
      commonFlowProps = new Props(commonFlowProps, flowParam);
    }
    flow.setInputProps(commonFlowProps);

    // Create execution dir
    createLogger(flowId);

    if (this.watcher != null) {
      this.watcher.setLogger(logger);
    }

    logger.info(
        "Running execid:"
            + execId
            + " flow:"
            + flowId
            + " project:"
            + projectId
            + " version:"
            + version);
    if (pipelineExecId != null) {
      logger.info(
          "Running simulateously with " + pipelineExecId + ". Pipelining level " + pipelineLevel);
    }

    // The current thread is used for interrupting blocks
    flowRunnerThread = Thread.currentThread();
    flowRunnerThread.setName("FlowRunner-exec-" + flow.getExecutionId());
  }
Exemple #4
0
  private JobRunner createJobRunner(ExecutableNode node, Props previousOutput) {
    String source = node.getJobPropsSource();
    String propsSource = node.getPropsSource();

    // If no properties are set, we just set the global properties.
    Props parentProps = propsSource == null ? globalProps : sharedProps.get(propsSource);

    // Set up overrides
    ExecutionOptions options = flow.getExecutionOptions();
    @SuppressWarnings("unchecked")
    Props flowProps = new Props(null, options.getFlowParameters());
    flowProps.putAll(commonProps);
    flowProps.setParent(parentProps);
    parentProps = flowProps;

    // We add the previous job output and put into this props.
    if (previousOutput != null) {
      Props earliestParent = previousOutput.getEarliestAncestor();
      earliestParent.setParent(parentProps);

      parentProps = previousOutput;
    }

    // Load job file.
    File path = new File(execDir, source);
    Props prop = null;

    // load the override props if any
    try {
      prop =
          projectLoader.fetchProjectProperty(
              flow.getProjectId(), flow.getVersion(), node.getJobId() + ".jor");
    } catch (ProjectManagerException e) {
      e.printStackTrace();
      logger.error("Error loading job override property for job " + node.getJobId());
    }
    if (prop == null) {
      // if no override prop, load the original one on disk
      try {
        prop = new Props(null, path);
      } catch (IOException e) {
        e.printStackTrace();
        logger.error("Error loading job file " + source + " for job " + node.getJobId());
      }
    }
    // setting this fake source as this will be used to determine the location of log files.
    prop.setSource(path.getPath());
    prop.setParent(parentProps);

    JobRunner jobRunner =
        new JobRunner(node, prop, path.getParentFile(), executorLoader, jobtypeManager);
    if (watcher != null) {
      jobRunner.setPipeline(watcher, pipelineLevel);
    }
    if (validateUserProxy) {
      jobRunner.setValidatedProxyUsers(proxyUsers);
    }

    jobRunner.setDelayStart(node.getDelayedExecution());
    jobRunner.setLogSettings(logger, jobLogFileSize, jobLogNumFiles);
    jobRunner.addListener(listener);

    return jobRunner;
  }