Пример #1
0
 /**
  * Executes a target flow. If execution is failed except on {@link ExecutionPhase#SETUP setup},
  * {@link ExecutionPhase#FINALIZE finalize}, or {@link ExecutionPhase#CLEANUP cleanup}, then the
  * {@code finalize} phase will be executed for recovery before execution is aborted.
  *
  * @param batchId target batch ID
  * @param flowId target flow ID
  * @param executionId target execution ID
  * @throws InterruptedException if interrupted during this execution
  * @throws IOException if failed to execute target flow
  * @throws IllegalArgumentException if some parameters were {@code null}
  */
 public void executeFlow(String batchId, String flowId, String executionId)
     throws InterruptedException, IOException {
   if (batchId == null) {
     throw new IllegalArgumentException("batchId must not be null"); // $NON-NLS-1$
   }
   if (flowId == null) {
     throw new IllegalArgumentException("flowId must not be null"); // $NON-NLS-1$
   }
   if (executionId == null) {
     throw new IllegalArgumentException("executionId must not be null"); // $NON-NLS-1$
   }
   FlowScript flow = script.findFlow(flowId);
   if (flow == null) {
     throw new IllegalArgumentException(
         MessageFormat.format(
             "Flow is undefined: batchId={0}, flowId={1}, executionId={2}",
             batchId, flowId, executionId));
   }
   ExecutionLock lock = acquireExecutionLock(batchId);
   try {
     lock.beginFlow(flowId, executionId);
     executeFlow(batchId, flow, executionId);
     lock.endFlow(flowId, executionId);
   } finally {
     lock.close();
   }
 }
Пример #2
0
  /**
   * Loads profile and create a new {@link ExecutionTask}.
   *
   * @param profile target profile
   * @param script target script
   * @param batchArguments current batch arguments
   * @param yaessArguments current controll arguments
   * @return the created task
   * @throws InterruptedException if interrupted while configuring services
   * @throws IOException if failed to configure services
   * @throws IllegalArgumentException if some parameters were {@code null}
   * @since 0.2.6
   */
  public static ExecutionTask load(
      YaessProfile profile,
      Properties script,
      Map<String, String> batchArguments,
      Map<String, String> yaessArguments)
      throws InterruptedException, IOException {
    if (profile == null) {
      throw new IllegalArgumentException("profile must not be null"); // $NON-NLS-1$
    }
    if (script == null) {
      throw new IllegalArgumentException("script must not be null"); // $NON-NLS-1$
    }
    if (batchArguments == null) {
      throw new IllegalArgumentException("batchArguments must not be null"); // $NON-NLS-1$
    }
    if (yaessArguments == null) {
      throw new IllegalArgumentException("yaessArguments must not be null"); // $NON-NLS-1$
    }
    LOG.debug("Loading execution monitor feature");
    ExecutionMonitorProvider monitors = profile.getMonitors().newInstance();

    LOG.debug("Loading execution lock feature");
    ExecutionLockProvider locks = profile.getLocks().newInstance();

    LOG.debug("Loading job scheduling feature");
    JobScheduler scheduler = profile.getScheduler().newInstance();

    LOG.debug("Loading hadoop execution feature");
    HadoopScriptHandler hadoopHandler = profile.getHadoopHandler().newInstance();

    LOG.debug("Loading command execution features");
    Map<String, CommandScriptHandler> commandHandlers = new HashMap<String, CommandScriptHandler>();
    for (Map.Entry<String, ServiceProfile<CommandScriptHandler>> entry :
        profile.getCommandHandlers().entrySet()) {
      commandHandlers.put(entry.getKey(), entry.getValue().newInstance());
    }

    LOG.debug("Extracting batch script");
    BatchScript batch = BatchScript.load(script);

    ExecutionTask result =
        new ExecutionTask(
            monitors, locks, scheduler, hadoopHandler, commandHandlers, batch, batchArguments);

    LOG.debug("Applying definitions");
    Map<String, String> copyDefinitions = new TreeMap<String, String>(yaessArguments);
    consumeRuntimeContext(result, copyDefinitions, batch);
    consumeSkipFlows(result, copyDefinitions, batch);
    consumeSerializeFlows(result, copyDefinitions, batch);
    checkRest(copyDefinitions);

    return result;
  }
Пример #3
0
  private static void consumeRuntimeContext(
      ExecutionTask result, Map<String, String> copyDefinitions, BatchScript script) {
    assert result != null;
    assert copyDefinitions != null;
    assert script != null;
    RuntimeContext rc = RuntimeContext.get().batchId(script.getId()).buildId(script.getBuildId());

    Ternary dryRunResult = consumeBoolean(copyDefinitions, KEY_VERIFY_DRYRUN);
    if (dryRunResult == Ternary.TRUE) {
      rc = rc.mode(ExecutionMode.SIMULATION);
    } else if (dryRunResult == Ternary.FALSE) {
      rc = rc.mode(ExecutionMode.PRODUCTION);
    }

    Ternary verify = consumeBoolean(copyDefinitions, KEY_VERIFY_APPLICATION);
    if (verify == Ternary.FALSE) {
      rc = rc.buildId(null);
    }

    result.runtimeContext = rc;
    result.getEnv().putAll(rc.unapply());
  }
Пример #4
0
 BatchScheduler(
     String batchId, BatchScript batchScript, ExecutionLock lock, ExecutorService executor) {
   assert batchId != null;
   assert batchScript != null;
   assert lock != null;
   assert executor != null;
   this.batchId = batchId;
   this.flows = new LinkedList<FlowScript>(batchScript.getAllFlows());
   this.lock = lock;
   this.executor = executor;
   this.running = new HashMap<String, FlowScriptTask>();
   this.blocking = new HashSet<String>();
   for (FlowScript flow : flows) {
     blocking.add(flow.getId());
   }
   this.doneQueue = new LinkedBlockingQueue<FlowScriptTask>();
 }
Пример #5
0
 /**
  * Executes a target phase.
  *
  * @param context the current context
  * @throws InterruptedException if interrupted during this execution
  * @throws IOException if failed to execute target phase
  * @throws IllegalArgumentException if some parameters were {@code null}
  * @since 0.2.5
  */
 public void executePhase(ExecutionContext context) throws InterruptedException, IOException {
   if (context == null) {
     throw new IllegalArgumentException("context must not be null"); // $NON-NLS-1$
   }
   FlowScript flow = script.findFlow(context.getFlowId());
   if (flow == null) {
     throw new IllegalArgumentException(
         MessageFormat.format(
             "Flow is undefined: batchId={0}, flowId={1}, executionId={2}",
             context.getBatchId(), context.getFlowId(), context.getExecutionId()));
   }
   Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase());
   ExecutionLock lock = acquireExecutionLock(context.getBatchId());
   try {
     lock.beginFlow(context.getFlowId(), context.getExecutionId());
     executePhase(context, executions);
     lock.endFlow(context.getFlowId(), context.getExecutionId());
   } finally {
     lock.close();
   }
 }
Пример #6
0
 private static void consumeSkipFlows(
     ExecutionTask task, Map<String, String> copyDefinitions, BatchScript script) {
   assert task != null;
   assert copyDefinitions != null;
   assert script != null;
   String flows = copyDefinitions.remove(KEY_SKIP_FLOWS);
   if (flows == null || flows.trim().isEmpty()) {
     return;
   }
   LOG.debug("Definition: {}={}", KEY_SKIP_FLOWS, flows);
   for (String flowIdCandidate : flows.split(",")) {
     String flowId = flowIdCandidate.trim();
     if (flowId.isEmpty() == false) {
       FlowScript flow = script.findFlow(flowId);
       if (flow == null) {
         throw new IllegalArgumentException(
             MessageFormat.format(
                 "Unknown flowId in definition {0} : {1}", KEY_SKIP_FLOWS, flowId));
       }
       task.skipFlows.add(flowId);
     }
   }
 }