示例#1
0
 private void submit(final FlowScript flow) {
   LOG.debug("Submitting jobflow \"{}\": {}", flow.getId(), batchId);
   FlowScriptTask task =
       new FlowScriptTask(
           flow,
           doneQueue,
           new Callable<Void>() {
             @Override
             public Void call() throws InterruptedException, IOException {
               if (Thread.interrupted()) {
                 throw new InterruptedException();
               }
               String executionId = UUID.randomUUID().toString();
               LOG.debug("Generated execution ID for \"{}\": {}", flow.getId(), executionId);
               lock.beginFlow(flow.getId(), executionId);
               executeFlow(batchId, flow, executionId);
               lock.endFlow(flow.getId(), executionId);
               LOG.debug("Completing jobflow \"{}\": {}", flow.getId(), batchId);
               return null;
             }
           });
   YSLOG.info("I01003", batchId, flow.getId());
   executor.execute(task);
   running.put(flow.getId(), task);
 }
示例#2
0
 private void waitForComplete() throws InterruptedException, IOException {
   LOG.debug("Waiting for running jobflows complete: {}", batchId);
   FlowScriptTask done = doneQueue.take();
   assert done.isDone();
   FlowScript flow = done.script;
   try {
     done.get();
     boolean blocked = blocking.remove(flow.getId());
     assert blocked;
   } catch (CancellationException e) {
     YSLOG.info(e, "I01005", batchId, flow.getId());
   } catch (ExecutionException e) {
     if (e.getCause() instanceof IOException) {
       throw (IOException) e.getCause();
     } else if (e.getCause() instanceof InterruptedException) {
       throw (InterruptedException) e.getCause();
     } else if (e.getCause() instanceof Error) {
       throw (Error) e.getCause();
     } else {
       throw new IOException("Flow execution failed by unknown error", e);
     }
   } finally {
     FlowScriptTask ran = running.remove(flow.getId());
     assert ran != null;
   }
 }
示例#3
0
 void executeFlow(String batchId, FlowScript flow, String executionId)
     throws InterruptedException, IOException {
   assert batchId != null;
   assert flow != null;
   assert executionId != null;
   YSLOG.info("I02000", batchId, flow.getId(), executionId);
   long start = System.currentTimeMillis();
   try {
     if (skipFlows.contains(flow.getId())) {
       YSLOG.info("I02002", batchId, flow.getId(), executionId);
       return;
     }
     executePhase(batchId, flow, executionId, ExecutionPhase.SETUP);
     boolean succeed = false;
     try {
       executePhase(batchId, flow, executionId, ExecutionPhase.INITIALIZE);
       executePhase(batchId, flow, executionId, ExecutionPhase.IMPORT);
       executePhase(batchId, flow, executionId, ExecutionPhase.PROLOGUE);
       executePhase(batchId, flow, executionId, ExecutionPhase.MAIN);
       executePhase(batchId, flow, executionId, ExecutionPhase.EPILOGUE);
       executePhase(batchId, flow, executionId, ExecutionPhase.EXPORT);
       succeed = true;
     } finally {
       if (succeed) {
         executePhase(batchId, flow, executionId, ExecutionPhase.FINALIZE);
       } else {
         YSLOG.info("I02003", batchId, flow.getId(), executionId);
         try {
           executePhase(batchId, flow, executionId, ExecutionPhase.FINALIZE);
         } catch (Exception e) {
           YSLOG.warn(e, "W02002", batchId, flow.getId(), executionId);
         }
       }
     }
     try {
       executePhase(batchId, flow, executionId, ExecutionPhase.CLEANUP);
     } catch (Exception e) {
       YSLOG.warn(e, "W02003", batchId, flow.getId(), executionId);
     }
     YSLOG.info("I02001", batchId, flow.getId(), executionId);
   } catch (IOException e) {
     YSLOG.error(e, "E02001", batchId, flow.getId(), executionId);
     throw e;
   } catch (InterruptedException e) {
     YSLOG.warn(e, "W02001", batchId, flow.getId(), executionId);
     throw e;
   } finally {
     long end = System.currentTimeMillis();
     YSLOG.info("I02999", batchId, flow.getId(), executionId, end - start);
   }
 }
示例#4
0
 private void executePhase(
     String batchId, FlowScript flow, String executionId, ExecutionPhase phase)
     throws InterruptedException, IOException {
   ExecutionContext context =
       new ExecutionContext(
           batchId,
           flow.getId(),
           executionId,
           phase,
           batchArguments,
           subprocessEnvironmentVaritables);
   Set<ExecutionScript> scripts = flow.getScripts().get(phase);
   assert scripts != null;
   executePhase(context, scripts);
 }
示例#5
0
 BatchScheduler(
     String batchId, BatchScript batchScript, ExecutionLock lock, ExecutorService executor) {
   assert batchId != null;
   assert batchScript != null;
   assert lock != null;
   assert executor != null;
   this.batchId = batchId;
   this.flows = new LinkedList<FlowScript>(batchScript.getAllFlows());
   this.lock = lock;
   this.executor = executor;
   this.running = new HashMap<String, FlowScriptTask>();
   this.blocking = new HashSet<String>();
   for (FlowScript flow : flows) {
     blocking.add(flow.getId());
   }
   this.doneQueue = new LinkedBlockingQueue<FlowScriptTask>();
 }
示例#6
0
 private boolean submit() {
   LOG.debug("Submitting waiting jobflows: {}", batchId);
   boolean submitted = false;
   for (Iterator<FlowScript> iter = flows.iterator(); iter.hasNext(); ) {
     FlowScript flow = iter.next();
     boolean blocked = false;
     for (String blockerId : flow.getBlockerIds()) {
       if (blocking.contains(blockerId)) {
         blocked = true;
         break;
       }
     }
     if (blocked == false) {
       submit(flow);
       iter.remove();
       submitted = true;
     }
   }
   return submitted;
 }
示例#7
0
 /**
  * Executes a target phase.
  *
  * @param context the current context
  * @throws InterruptedException if interrupted during this execution
  * @throws IOException if failed to execute target phase
  * @throws IllegalArgumentException if some parameters were {@code null}
  * @since 0.2.5
  */
 public void executePhase(ExecutionContext context) throws InterruptedException, IOException {
   if (context == null) {
     throw new IllegalArgumentException("context must not be null"); // $NON-NLS-1$
   }
   FlowScript flow = script.findFlow(context.getFlowId());
   if (flow == null) {
     throw new IllegalArgumentException(
         MessageFormat.format(
             "Flow is undefined: batchId={0}, flowId={1}, executionId={2}",
             context.getBatchId(), context.getFlowId(), context.getExecutionId()));
   }
   Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase());
   ExecutionLock lock = acquireExecutionLock(context.getBatchId());
   try {
     lock.beginFlow(context.getFlowId(), context.getExecutionId());
     executePhase(context, executions);
     lock.endFlow(context.getFlowId(), context.getExecutionId());
   } finally {
     lock.close();
   }
 }