public void testConfigNotPropagation() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set("abc", "xyz");
    workflow.setConf(newConf.toXmlString());

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");

    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });

    assertEquals(
        WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());

    subWorkflow.check(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.DONE, action.getStatus());

    subWorkflow.end(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
    assertNull(childConf.get("abc"));
  }
  public void testSubworkflowLib() throws Exception {
    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    FileSystem fs = getFileSystem();
    Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
    fs.create(parentLibJar);
    assertTrue(fs.exists(parentLibJar));
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
    workflow.setConf(newConf.toXmlString());

    Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();
    Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
    fs.create(subwfLibJar);
    assertTrue(fs.exists(subwfLibJar));

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "</sub-workflow>");
    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });

    assertEquals(
        WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
    subWorkflow.check(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.DONE, action.getStatus());
    subWorkflow.end(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
    childConf = wps.createProtoActionConf(childConf, "authToken", true);
    assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
  }
Esempio n. 3
0
  /**
   * Provides functionality to test kill node message
   *
   * @throws Exception
   */
  public void testKillNodeErrorMessage() throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-test-kill-node-message.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", "end.error");
    conf.set("external-status", "FAILED/KILLED");
    conf.set("signal-value", "fail");

    final String jobId = engine.submitJob(conf, true);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);

    waitFor(
        50000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
            return (job.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
          }
        });

    WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.KILLED, job.getStatus());

    WorkflowActionsGetForJobJPAExecutor wfActionsGetCmd =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actions = jpaService.execute(wfActionsGetCmd);

    int n = actions.size();
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
      if (bean.getType().equals("test")) {
        action = bean;
        break;
      }
    }
    assertNotNull(action);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals("end", action.getErrorMessage());
    assertEquals(WorkflowAction.Status.ERROR, action.getStatus());
  }
  private void _testUpdateAction(String actionId) throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);

    WorkflowActionGetJPAExecutor actionGetCmd = new WorkflowActionGetJPAExecutor(actionId);
    WorkflowActionBean action1 = jpaService.execute(actionGetCmd);
    action1.setStatus(WorkflowAction.Status.RUNNING);
    WorkflowActionUpdateJPAExecutor wfActionUpdateCmd =
        new WorkflowActionUpdateJPAExecutor(action1);
    jpaService.execute(wfActionUpdateCmd);

    WorkflowActionBean action2 = jpaService.execute(actionGetCmd);
    assertEquals(action2.getStatus(), WorkflowAction.Status.RUNNING);
  }
  public void testExecuteUpdate() throws Exception {

    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean bean =
        addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);
    bean.setStatus(WorkflowAction.Status.RUNNING);
    bean.setName("test-name");
    WorkflowActionQueryExecutor.getInstance()
        .executeUpdate(WorkflowActionQuery.UPDATE_ACTION, bean);
    WorkflowActionBean retBean =
        WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION, bean.getId());
    assertEquals("test-name", retBean.getName());
    assertEquals(retBean.getStatus(), WorkflowAction.Status.RUNNING);
  }
Esempio n. 6
0
  protected Void call(WorkflowStore store) throws StoreException, CommandException {
    WorkflowJobBean workflow = store.getWorkflow(jobId, false);
    setLogInfo(workflow);
    WorkflowActionBean action = store.getAction(id, false);
    setLogInfo(action);
    if (action.isPending()
        && (action.getStatus() == WorkflowActionBean.Status.DONE
            || action.getStatus() == WorkflowActionBean.Status.END_RETRY
            || action.getStatus() == WorkflowActionBean.Status.END_MANUAL)) {
      if (workflow.getStatus() == WorkflowJob.Status.RUNNING) {

        ActionExecutor executor =
            Services.get().get(ActionService.class).getExecutor(action.getType());
        Configuration conf = workflow.getWorkflowInstance().getConf();
        int maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries());
        long retryInterval =
            conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval());
        executor.setMaxRetries(maxRetries);
        executor.setRetryInterval(retryInterval);

        if (executor != null) {
          boolean isRetry = false;
          if (action.getStatus() == WorkflowActionBean.Status.END_RETRY
              || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) {
            isRetry = true;
          }
          ActionExecutorContext context =
              new ActionCommand.ActionExecutorContext(workflow, action, isRetry);
          try {

            XLog.getLog(getClass())
                .debug(
                    "End, name [{0}] type [{1}] status[{2}] external status [{3}] signal value [{4}]",
                    action.getName(),
                    action.getType(),
                    action.getStatus(),
                    action.getExternalStatus(),
                    action.getSignalValue());
            WorkflowInstance wfInstance = workflow.getWorkflowInstance();
            DagELFunctions.setActionInfo(wfInstance, action);
            workflow.setWorkflowInstance(wfInstance);
            incrActionCounter(action.getType(), 1);

            Instrumentation.Cron cron = new Instrumentation.Cron();
            cron.start();
            executor.end(context, action);
            cron.stop();
            addActionCron(action.getType(), cron);

            if (!context.isEnded()) {
              XLog.getLog(getClass())
                  .warn(
                      XLog.OPS,
                      "Action Ended, ActionExecutor [{0}] must call setEndData()",
                      executor.getType());
              action.setErrorInfo(
                  END_DATA_MISSING, "Execution Ended, but End Data Missing from Action");
              failJob(context);
              store.updateAction(action);
              store.updateWorkflow(workflow);
              return null;
            }
            action.setRetries(0);
            action.setEndTime(new Date());
            store.updateAction(action);
            store.updateWorkflow(workflow);
            Status slaStatus = null;
            switch (action.getStatus()) {
              case OK:
                slaStatus = Status.SUCCEEDED;
                break;
              case KILLED:
                slaStatus = Status.KILLED;
                break;
              case FAILED:
                slaStatus = Status.FAILED;
                break;
              case ERROR:
                XLog.getLog(getClass()).info("ERROR is considered as FAILED for SLA");
                slaStatus = Status.KILLED;
                break;
              default: // TODO: What will happen for other Action
                // status
                slaStatus = Status.FAILED;
                break;
            }
            SLADbOperations.writeStausEvent(
                action.getSlaXml(), action.getId(), store, slaStatus, SlaAppType.WORKFLOW_ACTION);
            queueCallable(new NotificationCommand(workflow, action));
            XLog.getLog(getClass())
                .debug(
                    "Queuing commands for action "
                        + id
                        + " status "
                        + action.getStatus()
                        + ", Set pending="
                        + action.getPending());
            queueCallable(new SignalCommand(workflow.getId(), id));
          } catch (ActionExecutorException ex) {
            XLog.getLog(getClass())
                .warn(
                    "Error ending action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]",
                    action.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage());
            action.setErrorInfo(ex.getErrorCode(), ex.getMessage());
            action.setEndTime(null);
            switch (ex.getErrorType()) {
              case TRANSIENT:
                if (!handleTransient(context, executor, WorkflowAction.Status.END_RETRY)) {
                  handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL);
                  action.setPendingAge(new Date());
                  action.setRetries(0);
                }
                action.setEndTime(null);
                break;
              case NON_TRANSIENT:
                handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL);
                action.setEndTime(null);
                break;
              case ERROR:
                handleError(context, executor, COULD_NOT_END, false, WorkflowAction.Status.ERROR);
                queueCallable(new SignalCommand(workflow.getId(), id));
                break;
              case FAILED:
                failJob(context);
                break;
            }
            store.updateAction(action);
            store.updateWorkflow(workflow);
          }
        } else {
          throw new CommandException(ErrorCode.E0802, action.getType());
        }
      } else {
        XLog.getLog(getClass())
            .warn(
                "Job state is not {0}. Skipping ActionEnd Execution",
                WorkflowJob.Status.RUNNING.toString());
      }
    } else {
      XLog.getLog(getClass())
          .debug(
              "Action pending={0}, status={1}. Skipping ActionEnd Execution",
              action.getPending(), action.getStatusStr());
    }
    return null;
  }
  public void testGetUpdateQuery() throws Exception {
    EntityManager em = jpaService.getEntityManager();
    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean bean =
        addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);

    // UPDATE_ACTION
    Query query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em);
    assertEquals(query.getParameterValue("conf"), bean.getConfBlob());
    assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("externalId"), bean.getExternalId());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("name"), bean.getName());
    assertEquals(query.getParameterValue("cred"), bean.getCred());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("type"), bean.getType());
    assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp());
    assertEquals(query.getParameterValue("executionPath"), bean.getExecutionPath());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("logToken"), bean.getLogToken());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAge());
    assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue());
    assertEquals(query.getParameterValue("slaXml"), bean.getSlaXmlBlob());
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("wfId"), bean.getWfId());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_FOR_LAST_CHECKED_TIME
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em);
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_PENDING
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_STATUS_PENDING
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_PENDING_TRANS
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_PENDING_TRANS_ERROR
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS_ERROR, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_START
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_START, bean, em);
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("conf"), bean.getConfBlob());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("externalId"), bean.getExternalId());
    assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri());
    assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_CHECK
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_CHECK, bean, em);
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_END
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_END, bean, em);
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("id"), bean.getId());
    em.close();
  }
Esempio n. 8
0
  public void testFsFailover() throws Exception {
    Reader reader = IOUtils.getResourceAsReader("failover-fs-wf.xml", -1);
    Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
    IOUtils.copyCharStream(reader, writer);

    final OozieClient wfClient = LocalOozie.getClient();
    Properties conf = wfClient.createConfiguration();
    conf.setProperty(OozieClient.APP_PATH, getTestCaseDir());
    conf.setProperty(OozieClient.USER_NAME, getTestUser());
    conf.setProperty(OozieClient.GROUP_NAME, getTestGroup());
    injectKerberosInfo(conf);
    final Path source = new Path(getFsTestCaseDir(), "fsfailover-source");
    getFileSystem().mkdirs(source);
    final Path target = new Path(getFsTestCaseDir().toString(), "fsfailover-target");
    conf.setProperty("source", source.toString());
    conf.setProperty("target", target.toUri().getPath());

    setSystemProperty(FaultInjection.FAULT_INJECTION, "true");
    setSystemProperty(SkipCommitFaultInjection.ACTION_FAILOVER_FAULT_INJECTION, "true");

    final String jobId1 = wfClient.submit(conf);
    wfClient.start(jobId1);

    waitFor(
        10 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return getFileSystem().exists(target);
          }
        });
    assertTrue(getFileSystem().exists(target));

    waitFor(
        10 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection");
          }
        });
    assertFalse(FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection"));

    assertEquals(WorkflowJob.Status.RUNNING, wfClient.getJobInfo(jobId1).getStatus());

    WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();

    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId1, false);
    WorkflowActionBean action = actions.get(0);
    assertEquals(WorkflowAction.Status.PREP, action.getStatus());

    setSystemProperty(FaultInjection.FAULT_INJECTION, "false");
    setSystemProperty(SkipCommitFaultInjection.ACTION_FAILOVER_FAULT_INJECTION, "false");

    ActionStartCommand actionStartCommand =
        new ActionStartCommand(action.getId(), action.getType());
    actionStartCommand.call();

    store = Services.get().get(WorkflowStoreService.class).create();
    actions = store.getActionsForWorkflow(jobId1, false);
    action = actions.get(0);
    assertEquals(WorkflowAction.Status.DONE, action.getStatus());

    waitFor(
        5 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.SUCCEEDED;
          }
        });
    assertEquals(WorkflowJob.Status.SUCCEEDED, wfClient.getJobInfo(jobId1).getStatus());

    final String jobId2 = wfClient.submit(conf);

    wfClient.start(jobId2);
    waitFor(
        10 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return wfClient.getJobInfo(jobId2).getStatus() == WorkflowJob.Status.KILLED;
          }
        });
    assertEquals(WorkflowJob.Status.KILLED, wfClient.getJobInfo(jobId2).getStatus());
  }
  public void testGetGroupFromParent() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    String actionConf =
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>";

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(actionConf);

    // negative test
    final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    workflow.setConf(newConf.toXmlString());

    subWorkflow.start(new Context(workflow, action), action);

    OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        5000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            subWorkflow.check(new Context(workflow, action), action);
            return action.getStatus() == WorkflowActionBean.Status.DONE;
          }
        });

    subWorkflow.check(new Context(workflow, action), action);
    subWorkflow.end(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));

    assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME));

    // positive test
    newConf.set(OozieClient.GROUP_NAME, getTestGroup());
    workflow.setConf(newConf.toXmlString());
    final WorkflowActionBean action1 = new WorkflowActionBean();
    action1.setConf(actionConf);
    action1.setId("W1");

    subWorkflow.start(new Context(workflow, action1), action1);

    oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL);

    waitFor(
        5000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            subWorkflow.check(new Context(workflow, action1), action1);
            return action1.getStatus() == WorkflowActionBean.Status.DONE;
          }
        });

    subWorkflow.check(new Context(workflow, action1), action1);
    subWorkflow.end(new Context(workflow, action1), action1);

    wf = oozieClient.getJobInfo(action1.getExternalId());
    childConf = new XConfiguration(new StringReader(wf.getConf()));
    assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME));
  }
  public void testSubWorkflowRecovery() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");

    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });
    String extId = action.getExternalId();
    assertEquals(WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(extId).getStatus());
    WorkflowActionBean action1 = new WorkflowActionBean();
    action1.setId(action.getId());
    action1.setName(action.getName());
    action1.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
            + "      <app-path>wrongAppPath</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");
    subWorkflow.start(new Context(workflow, action1), action1);
    assertEquals(extId, action1.getExternalId());
    subWorkflow.check(new Context(workflow, action1), action1);
    assertEquals(WorkflowAction.Status.DONE, action1.getStatus());
    subWorkflow.end(new Context(workflow, action1), action1);
    assertEquals(WorkflowAction.Status.OK, action1.getStatus());
  }
Esempio n. 11
0
  /**
   * Provides functionality to test transient failures.
   *
   * @param errorType the error type. (start.transient, end.transient)
   * @param expStatus1 expected status after the first step (START_RETRY, END_RETRY)
   * @param expStatus2 expected status after the second step (START_MANUAL, END_MANUAL)
   * @param expErrorMsg the expected error message.
   * @throws Exception
   */
  private void _testTransient(
      String errorType,
      WorkflowActionBean.Status expStatus1,
      final WorkflowActionBean.Status expStatus2,
      String expErrorMsg)
      throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final int maxRetries = 2;
    final int retryInterval = 10;

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);
    conf.setInt(OozieClient.ACTION_MAX_RETRIES, maxRetries);
    conf.setInt(OozieClient.ACTION_RETRY_INTERVAL, retryInterval);

    final String jobId = engine.submitJob(conf, true);

    int retryCount = 1;
    WorkflowActionBean.Status expectedStatus = expStatus1;
    int expectedRetryCount = 2;

    Thread.sleep(20000);
    String aId = null;
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    while (retryCount <= maxRetries) {
      List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false);
      WorkflowActionBean action = null;
      for (WorkflowActionBean bean : actions) {
        if (bean.getType().equals("test")) {
          action = bean;
          break;
        }
      }
      assertNotNull(action);
      aId = action.getId();
      assertEquals(expectedStatus, action.getStatus());
      assertEquals(expectedRetryCount, action.getRetries());
      assertEquals("TEST_ERROR", action.getErrorCode());
      assertEquals(expErrorMsg, action.getErrorMessage());
      if (action.getRetries() == maxRetries) {
        expectedRetryCount = 0;
        expectedStatus = expStatus2;
        break;
      } else {
        expectedRetryCount++;
      }
      Thread.sleep(retryInterval * 1000);
      retryCount++;
    }
    store.commitTrx();
    store.closeTrx();
    Thread.sleep(5000);

    final String actionId = aId;

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getWorkflowAction(actionId).getStatus() == expStatus2);
          }
        });

    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    store2.beginTrx();
    WorkflowActionBean action = engine.getWorkflowAction(actionId);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus2, action.getStatus());
    assertTrue(action.isPending() == false);
    assertEquals(WorkflowJob.Status.SUSPENDED, engine.getJob(jobId).getStatus());
    store2.commitTrx();
    store2.closeTrx();
  }
Esempio n. 12
0
  /**
   * Provides functionality to test non transient failures and coordinator action update
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
   * @param expErrorMsg expected error message.
   * @throws Exception
   */
  private void _testNonTransientWithCoordActionUpdate(
      String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);

    final String jobId = engine.submitJob(conf, false);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final CoordinatorJobBean coordJob =
        addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, false, false);
    CoordinatorActionBean coordAction =
        addRecordToCoordActionTable(
            coordJob.getId(),
            1,
            CoordinatorAction.Status.RUNNING,
            "coord-action-get.xml",
            jobId,
            "RUNNING",
            0);

    engine.start(jobId);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
          }
        });

    assertNotNull(jpaService);
    WorkflowJobGetJPAExecutor wfGetCmd = new WorkflowJobGetJPAExecutor(jobId);
    WorkflowJobBean job = jpaService.execute(wfGetCmd);
    WorkflowActionsGetForJobJPAExecutor actionsGetExe =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actionsList = jpaService.execute(actionsGetExe);

    int n = actionsList.size();
    WorkflowActionBean action = actionsList.get(n - 1);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus1, action.getStatus());
    assertFalse(action.isPending());

    assertEquals(WorkflowJob.Status.SUSPENDED, job.getStatus());

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            CoordinatorActionBean coordAction2 =
                jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId));
            return coordAction2.getStatus().equals(CoordinatorAction.Status.SUSPENDED);
          }
        });

    coordAction = jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId));
    assertEquals(CoordinatorAction.Status.SUSPENDED, coordAction.getStatus());
  }
Esempio n. 13
0
  /**
   * Provides functionality to test non transient failures.
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
   * @param expErrorMsg expected error message.
   * @throws Exception
   */
  private void _testNonTransient(
      String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);

    final String jobId = engine.submitJob(conf, true);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
          }
        });

    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, true);
    int n = actions.size();
    WorkflowActionBean action = actions.get(n - 1);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus1, action.getStatus());
    assertTrue(action.isPending() == false);

    assertTrue(engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);

    String actionConf = action.getConf();
    String fixedActionConf = actionConf.replaceAll(errorType, "none");
    action.setConf(fixedActionConf);
    store.updateAction(action);
    store.commitTrx();
    store.closeTrx();

    engine.resume(jobId);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED);
          }
        });

    assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());

    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    store2.beginTrx();
    actions = store2.getActionsForWorkflow(jobId, false);
    action = actions.get(0);
    assertEquals(null, action.getErrorCode());
    assertEquals(null, action.getErrorMessage());
    assertEquals(WorkflowActionBean.Status.OK, action.getStatus());
    store2.commitTrx();
    store2.closeTrx();
  }