コード例 #1
0
ファイル: FlowRunner.java プロジェクト: kakakakakku/azkaban2
 public void kill(String user) {
   synchronized (mainSyncObj) {
     logger.info("Flow killed by " + user);
     flow.setStatus(Status.KILLED);
     kill();
     updateFlow();
   }
   interrupt();
 }
コード例 #2
0
ファイル: FlowRunner.java プロジェクト: nielubowicz/azkaban2
  public void resume(String user) {
    synchronized (mainSyncObj) {
      if (!flowPaused) {
        logger.info("Cannot resume flow that isn't paused");
      } else {
        logger.info("Flow resumed by " + user);
        flowPaused = false;
        if (flowFailed) {
          flow.setStatus(Status.FAILED_FINISHING);
        } else if (flowCancelled) {
          flow.setStatus(Status.KILLED);
        } else {
          flow.setStatus(Status.RUNNING);
        }

        updateFlow();
      }
    }
  }
コード例 #3
0
ファイル: FlowRunner.java プロジェクト: nielubowicz/azkaban2
  public void pause(String user) {
    synchronized (mainSyncObj) {
      if (!flowFinished) {
        logger.info("Flow paused by " + user);
        flowPaused = true;
        flow.setStatus(Status.PAUSED);

        updateFlow();
      } else {
        logger.info("Cannot pause finished flow. Called by user " + user);
      }
    }

    interrupt();
  }
コード例 #4
0
ファイル: FlowRunner.java プロジェクト: nielubowicz/azkaban2
  public void retryJobs(List<String> jobIds, String user) {
    synchronized (mainSyncObj) {
      for (String jobId : jobIds) {
        ExecutableNode node = flow.getExecutableNode(jobId);
        if (node == null) {
          logger.error(
              "Job "
                  + jobId
                  + " doesn't exist in execution "
                  + flow.getExecutionId()
                  + ". Cannot retry.");
          continue;
        }

        if (Status.isStatusFinished(node.getStatus())) {
          // Resets the status and increments the attempt number
          node.resetForRetry();
          reEnableDependents(node);
          logger.info("Re-enabling job " + node.getJobId() + " attempt " + node.getAttempt());
        } else {
          logger.error("Cannot retry job " + jobId + " since it hasn't run yet. User " + user);
          continue;
        }
      }

      boolean isFailureFound = false;
      for (ExecutableNode node : flow.getExecutableNodes()) {
        Status nodeStatus = node.getStatus();
        if (nodeStatus == Status.FAILED || nodeStatus == Status.KILLED) {
          isFailureFound = true;
          break;
        }
      }

      if (!isFailureFound) {
        flow.setStatus(Status.RUNNING);
        flow.setUpdateTime(System.currentTimeMillis());
        flowFailed = false;
      }

      updateFlow();
      interrupt();
    }
  }
コード例 #5
0
ファイル: FlowRunner.java プロジェクト: nielubowicz/azkaban2
  private void cancel() {
    synchronized (mainSyncObj) {
      logger.info("Cancel has been called on flow " + execId);
      flowPaused = false;
      flowCancelled = true;

      if (watcher != null) {
        logger.info("Watcher is attached. Stopping watcher.");
        watcher.stopWatcher();
        logger.info("Watcher cancelled status is " + watcher.isWatchCancelled());
      }

      logger.info("Cancelling " + activeJobRunners.size() + " jobs.");
      for (JobRunner runner : activeJobRunners.values()) {
        runner.cancel();
      }

      if (flow.getStatus() != Status.FAILED && flow.getStatus() != Status.FAILED_FINISHING) {
        logger.info("Setting flow status to " + Status.KILLED.toString());
        flow.setStatus(Status.KILLED);
      }
    }
  }
コード例 #6
0
ファイル: FlowRunner.java プロジェクト: nielubowicz/azkaban2
  public void run() {
    try {
      if (this.executorService == null) {
        this.executorService = Executors.newFixedThreadPool(numJobThreads);
      }
      setupFlowExecution();
      flow.setStartTime(System.currentTimeMillis());

      updateFlowReference();

      logger.info("Updating initial flow directory.");
      updateFlow();
      logger.info("Fetching job and shared properties.");
      loadAllProperties();

      this.fireEventListeners(Event.create(this, Type.FLOW_STARTED));
      runFlow();
    } catch (Throwable t) {
      if (logger != null) {
        logger.error("An error has occurred during the running of the flow. Quiting.", t);
      }
      flow.setStatus(Status.FAILED);
    } finally {
      if (watcher != null) {
        logger.info("Watcher is attached. Stopping watcher.");
        watcher.stopWatcher();
        logger.info("Watcher cancelled status is " + watcher.isWatchCancelled());
      }

      flow.setEndTime(System.currentTimeMillis());
      logger.info("Setting end time for flow " + execId + " to " + System.currentTimeMillis());
      closeLogger();

      updateFlow();
      this.fireEventListeners(Event.create(this, Type.FLOW_FINISHED));
    }
  }
コード例 #7
0
ファイル: FlowRunner.java プロジェクト: nielubowicz/azkaban2
  /**
   * Main method that executes the jobs.
   *
   * @throws Exception
   */
  private void runFlow() throws Exception {
    logger.info("Starting flows");
    flow.setStatus(Status.RUNNING);
    updateFlow();

    while (!flowFinished) {
      synchronized (mainSyncObj) {
        if (flowPaused) {
          try {
            mainSyncObj.wait(CHECK_WAIT_MS);
          } catch (InterruptedException e) {
          }

          continue;
        } else {
          List<ExecutableNode> jobsReadyToRun = findReadyJobsToRun();

          if (!jobsReadyToRun.isEmpty() && !flowCancelled) {
            for (ExecutableNode node : jobsReadyToRun) {
              long currentTime = System.currentTimeMillis();

              // Queue a job only if it's ready to run.
              if (node.getStatus() == Status.READY) {
                // Collect output props from the job's dependencies.
                Props outputProps = collectOutputProps(node);
                node.setStatus(Status.QUEUED);
                JobRunner runner = createJobRunner(node, outputProps);
                logger.info("Submitting job " + node.getJobId() + " to run.");
                try {
                  executorService.submit(runner);
                  jobRunners.put(node.getJobId(), runner);
                  activeJobRunners.put(node.getJobId(), runner);
                } catch (RejectedExecutionException e) {
                  logger.error(e);
                }
                ;

              } // If killed, then auto complete and KILL
              else if (node.getStatus() == Status.KILLED) {
                logger.info("Killing " + node.getJobId() + " due to prior errors.");
                node.setStartTime(currentTime);
                node.setEndTime(currentTime);
                fireEventListeners(Event.create(this, Type.JOB_FINISHED, node));
              } // If disabled, then we auto skip
              else if (node.getStatus() == Status.DISABLED) {
                logger.info("Skipping disabled job " + node.getJobId() + ".");
                node.setStartTime(currentTime);
                node.setEndTime(currentTime);
                node.setStatus(Status.SKIPPED);
                fireEventListeners(Event.create(this, Type.JOB_FINISHED, node));
              }
            }

            updateFlow();
          } else {
            if (isFlowFinished() || flowCancelled) {
              flowFinished = true;
              break;
            }

            try {
              mainSyncObj.wait(CHECK_WAIT_MS);
            } catch (InterruptedException e) {
            }
          }
        }
      }
    }

    if (flowCancelled) {
      try {
        logger.info("Flow was force cancelled cleaning up.");
        for (JobRunner activeRunner : activeJobRunners.values()) {
          activeRunner.cancel();
        }

        for (ExecutableNode node : flow.getExecutableNodes()) {
          if (Status.isStatusFinished(node.getStatus())) {
            continue;
          } else if (node.getStatus() == Status.DISABLED) {
            node.setStatus(Status.SKIPPED);
          } else {
            node.setStatus(Status.KILLED);
          }
          fireEventListeners(Event.create(this, Type.JOB_FINISHED, node));
        }
      } catch (Exception e) {
        logger.error(e);
      }

      updateFlow();
    }

    logger.info("Finishing up flow. Awaiting Termination");
    executorService.shutdown();

    synchronized (mainSyncObj) {
      switch (flow.getStatus()) {
        case FAILED_FINISHING:
          logger.info("Setting flow status to Failed.");
          flow.setStatus(Status.FAILED);
        case FAILED:
        case KILLED:
          logger.info("Flow is set to " + flow.getStatus().toString());
          break;
        default:
          flow.setStatus(Status.SUCCEEDED);
          logger.info("Flow is set to " + flow.getStatus().toString());
      }
    }
  }