Beispiel #1
0
 @Override
 protected void verifyPrecondition() throws CommandException, PreconditionException {
   if ((wfAction == null) || (wfAction.isComplete() && wfAction.isPending())) {
     if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
         && wfJob.getStatus() != WorkflowJob.Status.PREP) {
       throw new PreconditionException(ErrorCode.E0813, wfJob.getStatusStr());
     }
   } else {
     throw new PreconditionException(
         ErrorCode.E0814, actionId, wfAction.getStatusStr(), wfAction.isPending());
   }
 }
  public void testActionCheckTransientDuringMRAction() throws Exception {
    // When using YARN, skip this test because it relies on shutting down the job tracker, which
    // isn't used in YARN
    if (createJobConf().get("yarn.resourcemanager.address") != null) {
      return;
    }
    services.destroy();
    // Make the max number of retries lower so the test won't take as long
    final int maxRetries = 2;
    setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
    services = new Services();
    // Disable ActionCheckerService so it doesn't interfere by triggering any extra
    // ActionCheckXCommands
    setClassesToBeExcluded(
        services.getConf(), new String[] {"org.apache.oozie.service.ActionCheckerService"});
    services.init();

    final JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job0 =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    final String jobId = job0.getId();
    WorkflowActionBean action0 =
        this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
    final String actionId = action0.getId();
    final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);

    new ActionStartXCommand(actionId, "map-reduce").call();
    final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
    String originalLauncherId = action1.getExternalId();

    ActionExecutorContext context =
        new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    JobConf conf =
        actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient =
        Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);

    final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
          }
        });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData =
        LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(action1.getId()).call();
    WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
    String originalMapperId = action2.getExternalChildIDs();

    assertFalse(originalLauncherId.equals(originalMapperId));

    // At this point, the launcher job has finished and the map-reduce action has started (but not
    // finished)
    // Now, shutdown the job tracker to pretend it has gone down during the map-reduce job
    executeWhileJobTrackerIsShutdown(
        new ShutdownJobTrackerExecutable() {
          @Override
          public void execute() throws Exception {
            assertEquals(0, action1.getRetries());
            new ActionCheckXCommand(actionId).call();

            waitFor(
                30 * 1000,
                new Predicate() {
                  @Override
                  public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() > 0);
                  }
                });
            waitFor(
                180 * 1000,
                new Predicate() {
                  @Override
                  public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() == 0);
                  }
                });
            WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
            assertEquals(0, action1b.getRetries());
            assertEquals("START_MANUAL", action1b.getStatusStr());

            WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
            assertEquals("SUSPENDED", job1.getStatusStr());

            // At this point, the action has gotten a transient error, even after maxRetries tries
            // so the workflow has been
            // SUSPENDED
          }
        });
    // Now, lets bring the job tracker back up and resume the workflow (which will restart the
    // current action)
    // It should now continue and finish with SUCCEEDED
    new ResumeXCommand(jobId).call();
    WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    assertEquals("RUNNING", job2.getStatusStr());

    sleep(500);

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
    String launcherId = action3.getExternalId();

    assertFalse(originalLauncherId.equals(launcherId));

    final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return launcherJob2.isComplete();
          }
        });

    assertTrue(launcherJob2.isSuccessful());
    actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
    String mapperId = action4.getExternalChildIDs();
    assertFalse(originalMapperId.equals(mapperId));

    final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return mrJob.isComplete();
          }
        });
    assertTrue(mrJob.isSuccessful());

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);

    assertEquals("SUCCEEDED", action5.getExternalStatus());
  }
Beispiel #3
0
  @Override
  protected Void execute() throws CommandException {
    LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
    WorkflowInstance workflowInstance = wfJob.getWorkflowInstance();
    workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob);
    boolean completed = false;
    boolean skipAction = false;
    if (wfAction == null) {
      if (wfJob.getStatus() == WorkflowJob.Status.PREP) {
        try {
          completed = workflowInstance.start();
        } catch (WorkflowException e) {
          throw new CommandException(e);
        }
        wfJob.setStatus(WorkflowJob.Status.RUNNING);
        wfJob.setStartTime(new Date());
        wfJob.setWorkflowInstance(workflowInstance);
        // 1. Add SLA status event for WF-JOB with status STARTED
        // 2. Add SLA registration events for all WF_ACTIONS
        SLADbXOperations.writeStausEvent(
            wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB);
        writeSLARegistrationForAllActions(
            workflowInstance.getApp().getDefinition(),
            wfJob.getUser(),
            wfJob.getGroup(),
            wfJob.getConf());
        queue(new NotificationXCommand(wfJob));
      } else {
        throw new CommandException(ErrorCode.E0801, wfJob.getId());
      }
    } else {
      String skipVar =
          workflowInstance.getVar(
              wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
      if (skipVar != null) {
        skipAction = skipVar.equals("true");
      }
      try {
        completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue());
      } catch (WorkflowException e) {
        throw new CommandException(e);
      }
      wfJob.setWorkflowInstance(workflowInstance);
      wfAction.resetPending();
      if (!skipAction) {
        wfAction.setTransition(workflowInstance.getTransition(wfAction.getName()));
      }
      try {
        jpaService.execute(new WorkflowActionUpdateJPAExecutor(wfAction));
      } catch (JPAExecutorException je) {
        throw new CommandException(je);
      }
    }

    if (completed) {
      try {
        for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) {
          WorkflowActionBean actionToKill;

          actionToKill = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToKillId));

          actionToKill.setPending();
          actionToKill.setStatus(WorkflowActionBean.Status.KILLED);
          jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToKill));
          queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType()));
        }

        for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) {
          WorkflowActionBean actionToFail =
              jpaService.execute(new WorkflowActionGetJPAExecutor(actionToFailId));
          actionToFail.resetPending();
          actionToFail.setStatus(WorkflowActionBean.Status.FAILED);
          SLADbXOperations.writeStausEvent(
              wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
          jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToFail));
        }
      } catch (JPAExecutorException je) {
        throw new CommandException(je);
      }

      wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
      wfJob.setEndTime(new Date());
      wfJob.setWorkflowInstance(workflowInstance);
      Status slaStatus = Status.SUCCEEDED;
      switch (wfJob.getStatus()) {
        case SUCCEEDED:
          slaStatus = Status.SUCCEEDED;
          break;
        case KILLED:
          slaStatus = Status.KILLED;
          break;
        case FAILED:
          slaStatus = Status.FAILED;
          break;
        default: // TODO SUSPENDED
          break;
      }
      SLADbXOperations.writeStausEvent(
          wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB);
      queue(new NotificationXCommand(wfJob));
      if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) {
        InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation());
      }
    } else {
      for (WorkflowActionBean newAction :
          WorkflowStoreService.getStartedActions(workflowInstance)) {
        String skipVar =
            workflowInstance.getVar(
                newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
        boolean skipNewAction = false;
        if (skipVar != null) {
          skipNewAction = skipVar.equals("true");
        }
        try {
          if (skipNewAction) {
            WorkflowActionBean oldAction;

            oldAction = jpaService.execute(new WorkflowActionGetJPAExecutor(newAction.getId()));

            oldAction.setPending();
            jpaService.execute(new WorkflowActionUpdateJPAExecutor(oldAction));

            queue(new SignalXCommand(jobId, oldAction.getId()));
          } else {
            newAction.setPending();
            String actionSlaXml =
                getActionSLAXml(
                    newAction.getName(),
                    workflowInstance.getApp().getDefinition(),
                    wfJob.getConf());
            newAction.setSlaXml(actionSlaXml);
            jpaService.execute(new WorkflowActionInsertJPAExecutor(newAction));
            LOG.debug(
                "SignalXCommand: Name: "
                    + newAction.getName()
                    + ", Id: "
                    + newAction.getId()
                    + ", Authcode:"
                    + newAction.getCred());
            queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
          }
        } catch (JPAExecutorException je) {
          throw new CommandException(je);
        }
      }
    }

    try {
      jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob));
    } catch (JPAExecutorException je) {
      throw new CommandException(je);
    }
    XLog.getLog(getClass())
        .debug(
            "Updated the workflow status to "
                + wfJob.getId()
                + "  status ="
                + wfJob.getStatusStr());
    if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
        && wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) {
      // update coordinator action
      new CoordActionUpdateXCommand(wfJob).call();
      new WfEndXCommand(wfJob).call(); // To delete the WF temp dir
    }
    LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
    return null;
  }
 protected void updateSLASummary() throws XException {
   updateSLASummary(wfJob.getId(), wfJob.getStartTime(), wfJob.getEndTime(), wfJob.getStatusStr());
 }