示例#1
0
  private void setupFlowExecution() {
    int projectId = flow.getProjectId();
    int version = flow.getVersion();
    String flowId = flow.getFlowId();

    // Add a bunch of common azkaban properties
    commonProps = PropsUtils.addCommonFlowProperties(flow);

    // Create execution dir
    createLogger(flowId);

    if (this.watcher != null) {
      this.watcher.setLogger(logger);
    }

    logger.info(
        "Running execid:"
            + execId
            + " flow:"
            + flowId
            + " project:"
            + projectId
            + " version:"
            + version);
    if (pipelineExecId != null) {
      logger.info(
          "Running simulateously with " + pipelineExecId + ". Pipelining level " + pipelineLevel);
    }

    // The current thread is used for interrupting blocks
    flowRunnerThread = Thread.currentThread();
    flowRunnerThread.setName("FlowRunner-exec-" + flow.getExecutionId());
  }
示例#2
0
  @SuppressWarnings("unchecked")
  private void setupFlowExecution() {
    int projectId = flow.getProjectId();
    int version = flow.getVersion();
    String flowId = flow.getFlowId();

    // Add a bunch of common azkaban properties
    Props commonFlowProps = PropsUtils.addCommonFlowProperties(null, flow);

    if (flow.getJobSource() != null) {
      String source = flow.getJobSource();
      Props flowProps = sharedProps.get(source);
      flowProps.setParent(commonFlowProps);
      commonFlowProps = flowProps;
    }

    // If there are flow overrides, we apply them now.
    Map<String, String> flowParam = flow.getExecutionOptions().getFlowParameters();
    if (flowParam != null && !flowParam.isEmpty()) {
      commonFlowProps = new Props(commonFlowProps, flowParam);
    }
    flow.setInputProps(commonFlowProps);

    // Create execution dir
    createLogger(flowId);

    if (this.watcher != null) {
      this.watcher.setLogger(logger);
    }

    logger.info(
        "Running execid:"
            + execId
            + " flow:"
            + flowId
            + " project:"
            + projectId
            + " version:"
            + version);
    if (pipelineExecId != null) {
      logger.info(
          "Running simulateously with " + pipelineExecId + ". Pipelining level " + pipelineLevel);
    }

    // The current thread is used for interrupting blocks
    flowRunnerThread = Thread.currentThread();
    flowRunnerThread.setName("FlowRunner-exec-" + flow.getExecutionId());
  }
示例#3
0
  public void retryJobs(List<String> jobIds, String user) {
    synchronized (mainSyncObj) {
      for (String jobId : jobIds) {
        ExecutableNode node = flow.getExecutableNode(jobId);
        if (node == null) {
          logger.error(
              "Job "
                  + jobId
                  + " doesn't exist in execution "
                  + flow.getExecutionId()
                  + ". Cannot retry.");
          continue;
        }

        if (Status.isStatusFinished(node.getStatus())) {
          // Resets the status and increments the attempt number
          node.resetForRetry();
          reEnableDependents(node);
          logger.info("Re-enabling job " + node.getJobId() + " attempt " + node.getAttempt());
        } else {
          logger.error("Cannot retry job " + jobId + " since it hasn't run yet. User " + user);
          continue;
        }
      }

      boolean isFailureFound = false;
      for (ExecutableNode node : flow.getExecutableNodes()) {
        Status nodeStatus = node.getStatus();
        if (nodeStatus == Status.FAILED || nodeStatus == Status.KILLED) {
          isFailureFound = true;
          break;
        }
      }

      if (!isFailureFound) {
        flow.setStatus(Status.RUNNING);
        flow.setUpdateTime(System.currentTimeMillis());
        flowFailed = false;
      }

      updateFlow();
      interrupt();
    }
  }
示例#4
0
  public FlowRunner(
      ExecutableFlow flow,
      ExecutorLoader executorLoader,
      ProjectLoader projectLoader,
      JobTypeManager jobtypeManager)
      throws ExecutorManagerException {
    this.execId = flow.getExecutionId();
    this.flow = flow;
    this.executorLoader = executorLoader;
    this.projectLoader = projectLoader;
    this.execDir = new File(flow.getExecutionPath());
    this.jobtypeManager = jobtypeManager;

    ExecutionOptions options = flow.getExecutionOptions();
    this.pipelineLevel = options.getPipelineLevel();
    this.pipelineExecId = options.getPipelineExecutionId();

    this.proxyUsers = flow.getProxyUsers();
  }
示例#5
0
  /**
   * Constructor. If executorService is null, then it will create it's own for thread pools.
   *
   * @param flow
   * @param executorLoader
   * @param projectLoader
   * @param jobtypeManager
   * @param executorService
   * @throws ExecutorManagerException
   */
  public FlowRunner(
      ExecutableFlow flow,
      ExecutorLoader executorLoader,
      ProjectLoader projectLoader,
      JobTypeManager jobtypeManager,
      ExecutorService executorService)
      throws ExecutorManagerException {
    this.execId = flow.getExecutionId();
    this.flow = flow;
    this.executorLoader = executorLoader;
    this.projectLoader = projectLoader;
    this.execDir = new File(flow.getExecutionPath());
    this.jobtypeManager = jobtypeManager;

    ExecutionOptions options = flow.getExecutionOptions();
    this.pipelineLevel = options.getPipelineLevel();
    this.pipelineExecId = options.getPipelineExecutionId();
    this.failureAction = options.getFailureAction();
    this.proxyUsers = flow.getProxyUsers();
    this.executorService = executorService;
    this.finishedNodes = new SwapQueue<ExecutableNode>();
  }