protected CoordinatorJobBean addRecordToCoordJobTable(
      CoordinatorJob.Status status, Date startTime, Date endTime, Date pauseTime, int timeout)
      throws Exception {
    CoordinatorJobBean coordJob = createCoordJob(status);
    coordJob.setStartTime(startTime);
    coordJob.setEndTime(endTime);
    coordJob.setPauseTime(pauseTime);
    coordJob.setFrequency(5);
    coordJob.setTimeUnit(Timeunit.MINUTE);
    coordJob.setTimeout(timeout);
    coordJob.setConcurrency(3);

    try {
      JPAService jpaService = Services.get().get(JPAService.class);
      assertNotNull(jpaService);
      CoordJobInsertJPAExecutor coordInsertCmd = new CoordJobInsertJPAExecutor(coordJob);
      jpaService.execute(coordInsertCmd);
    } catch (JPAExecutorException ex) {
      ex.printStackTrace();
      fail("Unable to insert the test coord job record to table");
      throw ex;
    }

    return coordJob;
  }
  private void checkCoordActions(String jobId, int number, CoordinatorJob.Status status) {
    try {
      JPAService jpaService = Services.get().get(JPAService.class);
      List<CoordinatorActionBean> actions =
          jpaService.execute(new CoordJobGetActionsJPAExecutor(jobId));
      if (actions.size() != number) {
        fail(
            "Should have "
                + number
                + " actions created for job "
                + jobId
                + ", but jave "
                + actions.size()
                + " actions.");
      }

      if (status != null) {
        CoordinatorJob job = jpaService.execute(new CoordJobGetJPAExecutor(jobId));
        if (job.getStatus() != status) {
          fail("Job status " + job.getStatus() + " should be " + status);
        }
      }
    } catch (JPAExecutorException se) {
      se.printStackTrace();
      fail("Job ID " + jobId + " was not stored properly in db");
    }
  }
  public void testActionCheck() throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean action =
        this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);
    WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(action.getId());

    new ActionStartXCommand(action.getId(), "map-reduce").call();
    action = jpaService.execute(wfActionGetCmd);

    ActionExecutorContext context =
        new ActionXCommand.ActionExecutorContext(job, action, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    JobConf conf =
        actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient =
        Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);

    String launcherId = action.getExternalId();

    final RunningJob launcherJob = jobClient.getJob(JobID.forName(launcherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
          }
        });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData =
        LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(action.getId()).call();
    action = jpaService.execute(wfActionGetCmd);
    String mapperId = action.getExternalId();
    String childId = action.getExternalChildIDs();

    assertTrue(launcherId.equals(mapperId));

    final RunningJob mrJob = jobClient.getJob(JobID.forName(childId));

    waitFor(
        120 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return mrJob.isComplete();
          }
        });
    assertTrue(mrJob.isSuccessful());

    new ActionCheckXCommand(action.getId()).call();
    action = jpaService.execute(wfActionGetCmd);

    assertEquals("SUCCEEDED", action.getExternalStatus());
  }
 private void checkCoordActionsTimeout(String actionId, int expected) {
   try {
     JPAService jpaService = Services.get().get(JPAService.class);
     CoordinatorActionBean action = jpaService.execute(new CoordActionGetJPAExecutor(actionId));
     assertEquals(action.getTimeOut(), expected);
   } catch (JPAExecutorException se) {
     se.printStackTrace();
     fail("Action ID " + actionId + " was not stored properly in db");
   }
 }
  private void _testGetJobInfoForId(String jobId) throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);
    Map<String, List<String>> filter = new HashMap<String, List<String>>();
    List<String> jobIdList = new ArrayList<String>();
    jobIdList.add(jobId);
    filter.put(OozieClient.FILTER_ID, jobIdList);

    BundleJobInfoGetJPAExecutor bundleInfoGetCmd = new BundleJobInfoGetJPAExecutor(filter, 1, 20);
    BundleJobInfo ret = jpaService.execute(bundleInfoGetCmd);
    assertNotNull(ret);
    assertEquals(ret.getBundleJobs().size(), 1);
  }
  private void _testGetJobInfoForAppName() throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);
    Map<String, List<String>> filter = new HashMap<String, List<String>>();
    List<String> list = new ArrayList<String>();
    list.add("BUNDLE-TEST");
    filter.put(OozieClient.FILTER_NAME, list);

    BundleJobInfoGetJPAExecutor bundleInfoGetCmd = new BundleJobInfoGetJPAExecutor(filter, 1, 20);
    BundleJobInfo ret = jpaService.execute(bundleInfoGetCmd);
    assertNotNull(ret);
    assertEquals(4, ret.getBundleJobs().size());
  }
Beispiel #7
0
  /**
   * Provides functionality to test user retry
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param externalStatus the external status to set.
   * @param signalValue the signal value to set.
   * @throws Exception
   */
  private void _testErrorWithUserRetry(String errorType, String externalStatus, String signalValue)
      throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid-user-retry.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", errorType);
    conf.set("external-status", externalStatus);
    conf.set("signal-value", signalValue);

    final String jobId = engine.submitJob(conf, true);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);

    final WorkflowActionsGetForJobJPAExecutor actionsGetExecutor =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor);
            WorkflowActionBean action = null;
            for (WorkflowActionBean bean : actions) {
              if (bean.getType().equals("test")) {
                action = bean;
                break;
              }
            }
            return (action != null && action.getUserRetryCount() == 2);
          }
        });

    List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor);
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
      if (bean.getType().equals("test")) {
        action = bean;
        break;
      }
    }
    assertNotNull(action);
    assertEquals(2, action.getUserRetryCount());
  }
Beispiel #8
0
  /**
   * Provides functionality to test kill node message
   *
   * @throws Exception
   */
  public void testKillNodeErrorMessage() throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-test-kill-node-message.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", "end.error");
    conf.set("external-status", "FAILED/KILLED");
    conf.set("signal-value", "fail");

    final String jobId = engine.submitJob(conf, true);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);

    waitFor(
        50000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
            return (job.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
          }
        });

    WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.KILLED, job.getStatus());

    WorkflowActionsGetForJobJPAExecutor wfActionsGetCmd =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actions = jpaService.execute(wfActionsGetCmd);

    int n = actions.size();
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
      if (bean.getType().equals("test")) {
        action = bean;
        break;
      }
    }
    assertNotNull(action);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals("end", action.getErrorMessage());
    assertEquals(WorkflowAction.Status.ERROR, action.getStatus());
  }
  private void _testUpdateAction(String actionId) throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);

    WorkflowActionGetJPAExecutor actionGetCmd = new WorkflowActionGetJPAExecutor(actionId);
    WorkflowActionBean action1 = jpaService.execute(actionGetCmd);
    action1.setStatus(WorkflowAction.Status.RUNNING);
    WorkflowActionUpdateJPAExecutor wfActionUpdateCmd =
        new WorkflowActionUpdateJPAExecutor(action1);
    jpaService.execute(wfActionUpdateCmd);

    WorkflowActionBean action2 = jpaService.execute(actionGetCmd);
    assertEquals(action2.getStatus(), WorkflowAction.Status.RUNNING);
  }
  public void testTimeout() throws Exception {
    Date startTime = DateUtils.parseDateUTC("2009-03-06T10:00Z");
    Date endTime = DateUtils.parseDateUTC("2009-03-06T10:14Z");
    Date pauseTime = null;
    CoordinatorJobBean job =
        addRecordToCoordJobTable(
            CoordinatorJob.Status.PREMATER, startTime, endTime, pauseTime, 300);
    new CoordActionMaterializeXCommand(job.getId(), startTime, endTime).call();

    JPAService jpaService = Services.get().get(JPAService.class);
    List<CoordinatorActionBean> actions =
        jpaService.execute(new CoordJobGetActionsJPAExecutor(job.getId()));
    checkCoordActionsTimeout(job.getId() + "@1", 300);
  }
  private CoordinatorActionBean checkCoordAction(String actionId) throws JPAExecutorException {
    long lastSeqId[] = new long[1];
    JPAService jpaService = Services.get().get(JPAService.class);
    List<SLAEventBean> slaEventList =
        jpaService.execute(new SLAEventsGetForSeqIdJPAExecutor(0, 10, lastSeqId));

    if (slaEventList.size() == 0) {
      fail("Unable to GET any record of sequence id greater than 0");
    }

    CoordinatorActionBean actionBean;
    actionBean = jpaService.execute(new CoordActionGetJPAExecutor(actionId));

    return actionBean;
  }
  /**
   * Test : Negative case - pause a suspended bundle
   *
   * @throws Exception
   */
  public void testBundlePauseUnpauseNeg1() throws Exception {
    BundleJobBean job = this.addRecordToBundleJobTable(Job.Status.SUSPENDED, false);

    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);
    BundleJobGetJPAExecutor bundleJobGetCmd = new BundleJobGetJPAExecutor(job.getId());
    job = jpaService.execute(bundleJobGetCmd);
    assertEquals(job.getStatus(), Job.Status.SUSPENDED);

    try {
      new BundlePauseXCommand(job).call();
      fail("should not reach here.");
    } catch (Exception ex) {
    }
  }
 @Override
 protected WorkflowActionBean addRecordToWfActionTable(
     String wfId, String actionName, WorkflowAction.Status status) throws Exception {
   WorkflowActionBean action = createWorkflowActionSetPending(wfId, status);
   try {
     JPAService jpaService = Services.get().get(JPAService.class);
     assertNotNull(jpaService);
     WorkflowActionInsertJPAExecutor actionInsertCmd = new WorkflowActionInsertJPAExecutor(action);
     jpaService.execute(actionInsertCmd);
   } catch (JPAExecutorException ce) {
     ce.printStackTrace();
     fail("Unable to insert the test wf action record to table");
     throw ce;
   }
   return action;
 }
  private void _testGetJobInfoForUserAndStatus() throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);
    Map<String, List<String>> filter = new HashMap<String, List<String>>();
    List<String> list1 = new ArrayList<String>();
    list1.add(getTestUser());
    filter.put(OozieClient.FILTER_USER, list1);
    List<String> list2 = new ArrayList<String>();
    list2.add("KILLED");
    filter.put(OozieClient.FILTER_STATUS, list2);

    BundleJobInfoGetJPAExecutor bundleInfoGetCmd = new BundleJobInfoGetJPAExecutor(filter, 1, 20);
    BundleJobInfo ret = jpaService.execute(bundleInfoGetCmd);
    assertNotNull(ret);
    assertEquals(1, ret.getBundleJobs().size());
  }
 /* (non-Javadoc)
  * @see org.apache.oozie.command.SuspendTransitionXCommand#performWrites()
  */
 @Override
 public void performWrites() throws CommandException {
   try {
     jpaService.execute(new BulkUpdateInsertJPAExecutor(updateList, null));
   } catch (JPAExecutorException e) {
     throw new CommandException(e);
   }
 }
Beispiel #16
0
 @Override
 protected void loadState() throws CommandException {
   try {
     jpaService = Services.get().get(JPAService.class);
     if (jpaService != null) {
       this.wfJob = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
       LogUtils.setLogInfo(wfJob, logInfo);
       if (actionId != null) {
         this.wfAction = jpaService.execute(new WorkflowActionGetJPAExecutor(actionId));
         LogUtils.setLogInfo(wfAction, logInfo);
       }
     } else {
       throw new CommandException(ErrorCode.E0610);
     }
   } catch (XException ex) {
     throw new CommandException(ex);
   }
 }
 /* (non-Javadoc)
  * @see org.apache.oozie.command.XCommand#loadState()
  */
 @Override
 protected void loadState() throws CommandException {
   try {
     eagerLoadState();
     this.bundleActions = jpaService.execute(new BundleActionsGetJPAExecutor(jobId));
   } catch (Exception Ex) {
     throw new CommandException(ErrorCode.E1311, this.jobId);
   }
 }
  /**
   * Test : Pause a RUNNINGWITHERROR bundle
   *
   * @throws Exception
   */
  public void testBundlePauseUnpause3() throws Exception {
    BundleJobBean job = this.addRecordToBundleJobTable(Job.Status.RUNNINGWITHERROR, false);

    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);
    BundleJobGetJPAExecutor bundleJobGetCmd = new BundleJobGetJPAExecutor(job.getId());
    job = jpaService.execute(bundleJobGetCmd);
    assertEquals(job.getStatus(), Job.Status.RUNNINGWITHERROR);

    new BundlePauseXCommand(job).call();

    job = jpaService.execute(bundleJobGetCmd);
    assertEquals(job.getStatus(), Job.Status.PAUSEDWITHERROR);

    new BundleUnpauseXCommand(job).call();
    job = jpaService.execute(bundleJobGetCmd);
    assertEquals(job.getStatus(), Job.Status.RUNNINGWITHERROR);
  }
  /* (non-Javadoc)
   * @see org.apache.oozie.command.XCommand#loadState()
   */
  @Override
  protected void loadState() throws CommandException {
    jpaService = Services.get().get(JPAService.class);
    if (jpaService == null) {
      throw new CommandException(ErrorCode.E0610);
    }

    try {
      bundleJob = jpaService.execute(new BundleJobGetJPAExecutor(jobId));
    } catch (Exception Ex) {
      throw new CommandException(ErrorCode.E0604, jobId);
    }

    try {
      bundleActions = jpaService.execute(new BundleActionsGetJPAExecutor(jobId));
    } catch (Exception Ex) {
      throw new CommandException(ErrorCode.E1311, jobId);
    }

    LogUtils.setLogInfo(bundleJob, logInfo);
  }
  /* (non-Javadoc)
   * @see org.apache.oozie.command.XCommand#eagerLoadState()
   */
  @Override
  protected void eagerLoadState() throws CommandException {
    try {
      jpaService = Services.get().get(JPAService.class);

      if (jpaService != null) {
        this.bundleJob = jpaService.execute(new BundleJobGetJPAExecutor(jobId));
        LogUtils.setLogInfo(bundleJob, logInfo);
      } else {
        throw new CommandException(ErrorCode.E0610);
      }
    } catch (XException ex) {
      throw new CommandException(ex);
    }
  }
 /* (non-Javadoc)
  * @see org.apache.oozie.command.XCommand#execute()
  */
 @Override
 protected Void execute() throws CommandException {
   try {
     if (isChangePauseTime || isChangeEndTime) {
       if (isChangePauseTime) {
         bundleJob.setPauseTime(newPauseTime);
       } else if (isChangeEndTime) {
         bundleJob.setEndTime(newEndTime);
         if (bundleJob.getStatus() == Job.Status.SUCCEEDED) {
           bundleJob.setStatus(Job.Status.RUNNING);
         }
         if (bundleJob.getStatus() == Job.Status.DONEWITHERROR
             || bundleJob.getStatus() == Job.Status.FAILED) {
           bundleJob.setStatus(
               StatusUtils.getStatusIfBackwardSupportTrue(Job.Status.RUNNINGWITHERROR));
         }
       }
       for (BundleActionBean action : this.bundleActions) {
         // queue coord change commands;
         if (action.getStatus() != Job.Status.KILLED && action.getCoordId() != null) {
           queue(new CoordChangeXCommand(action.getCoordId(), changeValue));
           LOG.info(
               "Queuing CoordChangeXCommand coord job = "
                   + action.getCoordId()
                   + " to change "
                   + changeValue);
           action.setPending(action.getPending() + 1);
           updateList.add(action);
         }
       }
       updateList.add(bundleJob);
       jpaService.execute(new BulkUpdateInsertJPAExecutor(updateList, null));
     }
     return null;
   } catch (XException ex) {
     throw new CommandException(ex);
   }
 }
Beispiel #22
0
  /**
   * Write a coordinator job into database
   *
   * @param eJob : XML element of job
   * @param coordJob : Coordinator job bean
   * @return Job id
   * @throws CommandException thrown if unable to save coordinator job to db
   */
  private String storeToDB(Element eJob, CoordinatorJobBean coordJob) throws CommandException {
    String jobId = Services.get().get(UUIDService.class).generateId(ApplicationType.COORDINATOR);
    coordJob.setId(jobId);
    coordJob.setAuthToken(this.authToken);

    coordJob.setAppPath(conf.get(OozieClient.COORDINATOR_APP_PATH));
    coordJob.setCreatedTime(new Date());
    coordJob.setUser(conf.get(OozieClient.USER_NAME));
    coordJob.setGroup(conf.get(OozieClient.GROUP_NAME));
    coordJob.setConf(XmlUtils.prettyPrint(conf).toString());
    coordJob.setJobXml(XmlUtils.prettyPrint(eJob).toString());
    coordJob.setLastActionNumber(0);
    coordJob.setLastModifiedTime(new Date());

    if (!dryrun) {
      coordJob.setLastModifiedTime(new Date());
      try {
        jpaService.execute(new CoordJobInsertJPAExecutor(coordJob));
      } catch (JPAExecutorException je) {
        throw new CommandException(je);
      }
    }
    return jobId;
  }
Beispiel #23
0
  /**
   * Provides functionality to test non transient failures and coordinator action update
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
   * @param expErrorMsg expected error message.
   * @throws Exception
   */
  private void _testNonTransientWithCoordActionUpdate(
      String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);

    final String jobId = engine.submitJob(conf, false);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final CoordinatorJobBean coordJob =
        addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, false, false);
    CoordinatorActionBean coordAction =
        addRecordToCoordActionTable(
            coordJob.getId(),
            1,
            CoordinatorAction.Status.RUNNING,
            "coord-action-get.xml",
            jobId,
            "RUNNING",
            0);

    engine.start(jobId);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
          }
        });

    assertNotNull(jpaService);
    WorkflowJobGetJPAExecutor wfGetCmd = new WorkflowJobGetJPAExecutor(jobId);
    WorkflowJobBean job = jpaService.execute(wfGetCmd);
    WorkflowActionsGetForJobJPAExecutor actionsGetExe =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actionsList = jpaService.execute(actionsGetExe);

    int n = actionsList.size();
    WorkflowActionBean action = actionsList.get(n - 1);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus1, action.getStatus());
    assertFalse(action.isPending());

    assertEquals(WorkflowJob.Status.SUSPENDED, job.getStatus());

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            CoordinatorActionBean coordAction2 =
                jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId));
            return coordAction2.getStatus().equals(CoordinatorAction.Status.SUSPENDED);
          }
        });

    coordAction = jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId));
    assertEquals(CoordinatorAction.Status.SUSPENDED, coordAction.getStatus());
  }
  public void testGetUpdateQuery() throws Exception {
    EntityManager em = jpaService.getEntityManager();
    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean bean =
        addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);

    // UPDATE_ACTION
    Query query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em);
    assertEquals(query.getParameterValue("conf"), bean.getConfBlob());
    assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("externalId"), bean.getExternalId());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("name"), bean.getName());
    assertEquals(query.getParameterValue("cred"), bean.getCred());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("type"), bean.getType());
    assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp());
    assertEquals(query.getParameterValue("executionPath"), bean.getExecutionPath());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("logToken"), bean.getLogToken());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAge());
    assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue());
    assertEquals(query.getParameterValue("slaXml"), bean.getSlaXmlBlob());
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("wfId"), bean.getWfId());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_FOR_LAST_CHECKED_TIME
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em);
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_PENDING
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_STATUS_PENDING
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_PENDING_TRANS
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_PENDING_TRANS_ERROR
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS_ERROR, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_START
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_START, bean, em);
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("conf"), bean.getConfBlob());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("externalId"), bean.getExternalId());
    assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri());
    assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_CHECK
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_CHECK, bean, em);
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_END
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_END, bean, em);
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("id"), bean.getId());
    em.close();
  }
  public void testActionCheckTransientDuringMRAction() throws Exception {
    // When using YARN, skip this test because it relies on shutting down the job tracker, which
    // isn't used in YARN
    if (createJobConf().get("yarn.resourcemanager.address") != null) {
      return;
    }
    services.destroy();
    // Make the max number of retries lower so the test won't take as long
    final int maxRetries = 2;
    setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
    services = new Services();
    // Disable ActionCheckerService so it doesn't interfere by triggering any extra
    // ActionCheckXCommands
    setClassesToBeExcluded(
        services.getConf(), new String[] {"org.apache.oozie.service.ActionCheckerService"});
    services.init();

    final JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job0 =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    final String jobId = job0.getId();
    WorkflowActionBean action0 =
        this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
    final String actionId = action0.getId();
    final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);

    new ActionStartXCommand(actionId, "map-reduce").call();
    final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
    String originalLauncherId = action1.getExternalId();

    ActionExecutorContext context =
        new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    JobConf conf =
        actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient =
        Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);

    final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
          }
        });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData =
        LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(action1.getId()).call();
    WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
    String originalMapperId = action2.getExternalChildIDs();

    assertFalse(originalLauncherId.equals(originalMapperId));

    // At this point, the launcher job has finished and the map-reduce action has started (but not
    // finished)
    // Now, shutdown the job tracker to pretend it has gone down during the map-reduce job
    executeWhileJobTrackerIsShutdown(
        new ShutdownJobTrackerExecutable() {
          @Override
          public void execute() throws Exception {
            assertEquals(0, action1.getRetries());
            new ActionCheckXCommand(actionId).call();

            waitFor(
                30 * 1000,
                new Predicate() {
                  @Override
                  public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() > 0);
                  }
                });
            waitFor(
                180 * 1000,
                new Predicate() {
                  @Override
                  public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() == 0);
                  }
                });
            WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
            assertEquals(0, action1b.getRetries());
            assertEquals("START_MANUAL", action1b.getStatusStr());

            WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
            assertEquals("SUSPENDED", job1.getStatusStr());

            // At this point, the action has gotten a transient error, even after maxRetries tries
            // so the workflow has been
            // SUSPENDED
          }
        });
    // Now, lets bring the job tracker back up and resume the workflow (which will restart the
    // current action)
    // It should now continue and finish with SUCCEEDED
    new ResumeXCommand(jobId).call();
    WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    assertEquals("RUNNING", job2.getStatusStr());

    sleep(500);

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
    String launcherId = action3.getExternalId();

    assertFalse(originalLauncherId.equals(launcherId));

    final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return launcherJob2.isComplete();
          }
        });

    assertTrue(launcherJob2.isSuccessful());
    actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
    String mapperId = action4.getExternalChildIDs();
    assertFalse(originalMapperId.equals(mapperId));

    final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return mrJob.isComplete();
          }
        });
    assertTrue(mrJob.isSuccessful());

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);

    assertEquals("SUCCEEDED", action5.getExternalStatus());
  }
Beispiel #26
0
  @Override
  protected Void execute() throws CommandException {
    LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
    WorkflowInstance workflowInstance = wfJob.getWorkflowInstance();
    workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob);
    boolean completed = false;
    boolean skipAction = false;
    if (wfAction == null) {
      if (wfJob.getStatus() == WorkflowJob.Status.PREP) {
        try {
          completed = workflowInstance.start();
        } catch (WorkflowException e) {
          throw new CommandException(e);
        }
        wfJob.setStatus(WorkflowJob.Status.RUNNING);
        wfJob.setStartTime(new Date());
        wfJob.setWorkflowInstance(workflowInstance);
        // 1. Add SLA status event for WF-JOB with status STARTED
        // 2. Add SLA registration events for all WF_ACTIONS
        SLADbXOperations.writeStausEvent(
            wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB);
        writeSLARegistrationForAllActions(
            workflowInstance.getApp().getDefinition(),
            wfJob.getUser(),
            wfJob.getGroup(),
            wfJob.getConf());
        queue(new NotificationXCommand(wfJob));
      } else {
        throw new CommandException(ErrorCode.E0801, wfJob.getId());
      }
    } else {
      String skipVar =
          workflowInstance.getVar(
              wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
      if (skipVar != null) {
        skipAction = skipVar.equals("true");
      }
      try {
        completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue());
      } catch (WorkflowException e) {
        throw new CommandException(e);
      }
      wfJob.setWorkflowInstance(workflowInstance);
      wfAction.resetPending();
      if (!skipAction) {
        wfAction.setTransition(workflowInstance.getTransition(wfAction.getName()));
      }
      try {
        jpaService.execute(new WorkflowActionUpdateJPAExecutor(wfAction));
      } catch (JPAExecutorException je) {
        throw new CommandException(je);
      }
    }

    if (completed) {
      try {
        for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) {
          WorkflowActionBean actionToKill;

          actionToKill = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToKillId));

          actionToKill.setPending();
          actionToKill.setStatus(WorkflowActionBean.Status.KILLED);
          jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToKill));
          queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType()));
        }

        for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) {
          WorkflowActionBean actionToFail =
              jpaService.execute(new WorkflowActionGetJPAExecutor(actionToFailId));
          actionToFail.resetPending();
          actionToFail.setStatus(WorkflowActionBean.Status.FAILED);
          SLADbXOperations.writeStausEvent(
              wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
          jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToFail));
        }
      } catch (JPAExecutorException je) {
        throw new CommandException(je);
      }

      wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
      wfJob.setEndTime(new Date());
      wfJob.setWorkflowInstance(workflowInstance);
      Status slaStatus = Status.SUCCEEDED;
      switch (wfJob.getStatus()) {
        case SUCCEEDED:
          slaStatus = Status.SUCCEEDED;
          break;
        case KILLED:
          slaStatus = Status.KILLED;
          break;
        case FAILED:
          slaStatus = Status.FAILED;
          break;
        default: // TODO SUSPENDED
          break;
      }
      SLADbXOperations.writeStausEvent(
          wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB);
      queue(new NotificationXCommand(wfJob));
      if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) {
        InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation());
      }
    } else {
      for (WorkflowActionBean newAction :
          WorkflowStoreService.getStartedActions(workflowInstance)) {
        String skipVar =
            workflowInstance.getVar(
                newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
        boolean skipNewAction = false;
        if (skipVar != null) {
          skipNewAction = skipVar.equals("true");
        }
        try {
          if (skipNewAction) {
            WorkflowActionBean oldAction;

            oldAction = jpaService.execute(new WorkflowActionGetJPAExecutor(newAction.getId()));

            oldAction.setPending();
            jpaService.execute(new WorkflowActionUpdateJPAExecutor(oldAction));

            queue(new SignalXCommand(jobId, oldAction.getId()));
          } else {
            newAction.setPending();
            String actionSlaXml =
                getActionSLAXml(
                    newAction.getName(),
                    workflowInstance.getApp().getDefinition(),
                    wfJob.getConf());
            newAction.setSlaXml(actionSlaXml);
            jpaService.execute(new WorkflowActionInsertJPAExecutor(newAction));
            LOG.debug(
                "SignalXCommand: Name: "
                    + newAction.getName()
                    + ", Id: "
                    + newAction.getId()
                    + ", Authcode:"
                    + newAction.getCred());
            queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
          }
        } catch (JPAExecutorException je) {
          throw new CommandException(je);
        }
      }
    }

    try {
      jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob));
    } catch (JPAExecutorException je) {
      throw new CommandException(je);
    }
    XLog.getLog(getClass())
        .debug(
            "Updated the workflow status to "
                + wfJob.getId()
                + "  status ="
                + wfJob.getStatusStr());
    if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
        && wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) {
      // update coordinator action
      new CoordActionUpdateXCommand(wfJob).call();
      new WfEndXCommand(wfJob).call(); // To delete the WF temp dir
    }
    LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
    return null;
  }