/** * Executes a target flow. If execution is failed except on {@link ExecutionPhase#SETUP setup}, * {@link ExecutionPhase#FINALIZE finalize}, or {@link ExecutionPhase#CLEANUP cleanup}, then the * {@code finalize} phase will be executed for recovery before execution is aborted. * * @param batchId target batch ID * @param flowId target flow ID * @param executionId target execution ID * @throws InterruptedException if interrupted during this execution * @throws IOException if failed to execute target flow * @throws IllegalArgumentException if some parameters were {@code null} */ public void executeFlow(String batchId, String flowId, String executionId) throws InterruptedException, IOException { if (batchId == null) { throw new IllegalArgumentException("batchId must not be null"); // $NON-NLS-1$ } if (flowId == null) { throw new IllegalArgumentException("flowId must not be null"); // $NON-NLS-1$ } if (executionId == null) { throw new IllegalArgumentException("executionId must not be null"); // $NON-NLS-1$ } FlowScript flow = script.findFlow(flowId); if (flow == null) { throw new IllegalArgumentException( MessageFormat.format( "Flow is undefined: batchId={0}, flowId={1}, executionId={2}", batchId, flowId, executionId)); } ExecutionLock lock = acquireExecutionLock(batchId); try { lock.beginFlow(flowId, executionId); executeFlow(batchId, flow, executionId); lock.endFlow(flowId, executionId); } finally { lock.close(); } }
/** * Loads profile and create a new {@link ExecutionTask}. * * @param profile target profile * @param script target script * @param batchArguments current batch arguments * @param yaessArguments current controll arguments * @return the created task * @throws InterruptedException if interrupted while configuring services * @throws IOException if failed to configure services * @throws IllegalArgumentException if some parameters were {@code null} * @since 0.2.6 */ public static ExecutionTask load( YaessProfile profile, Properties script, Map<String, String> batchArguments, Map<String, String> yaessArguments) throws InterruptedException, IOException { if (profile == null) { throw new IllegalArgumentException("profile must not be null"); // $NON-NLS-1$ } if (script == null) { throw new IllegalArgumentException("script must not be null"); // $NON-NLS-1$ } if (batchArguments == null) { throw new IllegalArgumentException("batchArguments must not be null"); // $NON-NLS-1$ } if (yaessArguments == null) { throw new IllegalArgumentException("yaessArguments must not be null"); // $NON-NLS-1$ } LOG.debug("Loading execution monitor feature"); ExecutionMonitorProvider monitors = profile.getMonitors().newInstance(); LOG.debug("Loading execution lock feature"); ExecutionLockProvider locks = profile.getLocks().newInstance(); LOG.debug("Loading job scheduling feature"); JobScheduler scheduler = profile.getScheduler().newInstance(); LOG.debug("Loading hadoop execution feature"); HadoopScriptHandler hadoopHandler = profile.getHadoopHandler().newInstance(); LOG.debug("Loading command execution features"); Map<String, CommandScriptHandler> commandHandlers = new HashMap<String, CommandScriptHandler>(); for (Map.Entry<String, ServiceProfile<CommandScriptHandler>> entry : profile.getCommandHandlers().entrySet()) { commandHandlers.put(entry.getKey(), entry.getValue().newInstance()); } LOG.debug("Extracting batch script"); BatchScript batch = BatchScript.load(script); ExecutionTask result = new ExecutionTask( monitors, locks, scheduler, hadoopHandler, commandHandlers, batch, batchArguments); LOG.debug("Applying definitions"); Map<String, String> copyDefinitions = new TreeMap<String, String>(yaessArguments); consumeRuntimeContext(result, copyDefinitions, batch); consumeSkipFlows(result, copyDefinitions, batch); consumeSerializeFlows(result, copyDefinitions, batch); checkRest(copyDefinitions); return result; }
private static void consumeRuntimeContext( ExecutionTask result, Map<String, String> copyDefinitions, BatchScript script) { assert result != null; assert copyDefinitions != null; assert script != null; RuntimeContext rc = RuntimeContext.get().batchId(script.getId()).buildId(script.getBuildId()); Ternary dryRunResult = consumeBoolean(copyDefinitions, KEY_VERIFY_DRYRUN); if (dryRunResult == Ternary.TRUE) { rc = rc.mode(ExecutionMode.SIMULATION); } else if (dryRunResult == Ternary.FALSE) { rc = rc.mode(ExecutionMode.PRODUCTION); } Ternary verify = consumeBoolean(copyDefinitions, KEY_VERIFY_APPLICATION); if (verify == Ternary.FALSE) { rc = rc.buildId(null); } result.runtimeContext = rc; result.getEnv().putAll(rc.unapply()); }
BatchScheduler( String batchId, BatchScript batchScript, ExecutionLock lock, ExecutorService executor) { assert batchId != null; assert batchScript != null; assert lock != null; assert executor != null; this.batchId = batchId; this.flows = new LinkedList<FlowScript>(batchScript.getAllFlows()); this.lock = lock; this.executor = executor; this.running = new HashMap<String, FlowScriptTask>(); this.blocking = new HashSet<String>(); for (FlowScript flow : flows) { blocking.add(flow.getId()); } this.doneQueue = new LinkedBlockingQueue<FlowScriptTask>(); }
/** * Executes a target phase. * * @param context the current context * @throws InterruptedException if interrupted during this execution * @throws IOException if failed to execute target phase * @throws IllegalArgumentException if some parameters were {@code null} * @since 0.2.5 */ public void executePhase(ExecutionContext context) throws InterruptedException, IOException { if (context == null) { throw new IllegalArgumentException("context must not be null"); // $NON-NLS-1$ } FlowScript flow = script.findFlow(context.getFlowId()); if (flow == null) { throw new IllegalArgumentException( MessageFormat.format( "Flow is undefined: batchId={0}, flowId={1}, executionId={2}", context.getBatchId(), context.getFlowId(), context.getExecutionId())); } Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase()); ExecutionLock lock = acquireExecutionLock(context.getBatchId()); try { lock.beginFlow(context.getFlowId(), context.getExecutionId()); executePhase(context, executions); lock.endFlow(context.getFlowId(), context.getExecutionId()); } finally { lock.close(); } }
private static void consumeSkipFlows( ExecutionTask task, Map<String, String> copyDefinitions, BatchScript script) { assert task != null; assert copyDefinitions != null; assert script != null; String flows = copyDefinitions.remove(KEY_SKIP_FLOWS); if (flows == null || flows.trim().isEmpty()) { return; } LOG.debug("Definition: {}={}", KEY_SKIP_FLOWS, flows); for (String flowIdCandidate : flows.split(",")) { String flowId = flowIdCandidate.trim(); if (flowId.isEmpty() == false) { FlowScript flow = script.findFlow(flowId); if (flow == null) { throw new IllegalArgumentException( MessageFormat.format( "Unknown flowId in definition {0} : {1}", KEY_SKIP_FLOWS, flowId)); } task.skipFlows.add(flowId); } } }