Exemplo n.º 1
0
  /**
   * Provides functionality to test errors
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param externalStatus the external status to set.
   * @param signalValue the signal value to set.
   * @throws Exception
   */
  private void _testError(String errorType, String externalStatus, String signalValue)
      throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", errorType);
    conf.set("external-status", externalStatus);
    conf.set("signal-value", signalValue);

    final String jobId = engine.submitJob(conf, true);

    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            WorkflowJobBean bean = store.getWorkflow(jobId, false);
            return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
          }
        });
    assertEquals(WorkflowJob.Status.KILLED, engine.getJob(jobId).getStatus());
    store.commitTrx();
    store.closeTrx();
  }
Exemplo n.º 2
0
  /**
   * v1 service implementation to get a list of workflows, with filtering or interested windows
   * embedded in the request object
   */
  private JSONObject getWorkflowJobs(HttpServletRequest request) throws XServletException {
    JSONObject json = new JSONObject();
    try {
      String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
      String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
      String lenStr = request.getParameter(RestConstants.LEN_PARAM);
      int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
      start = (start < 1) ? 1 : start;
      int len = (lenStr != null) ? Integer.parseInt(lenStr) : 50;
      len = (len < 1) ? 50 : len;
      DagEngine dagEngine =
          Services.get()
              .get(DagEngineService.class)
              .getDagEngine(getUser(request), getAuthToken(request));
      WorkflowsInfo jobs = dagEngine.getJobs(filter, start, len);
      List<WorkflowJobBean> jsonWorkflows = jobs.getWorkflows();
      json.put(JsonTags.WORKFLOWS_JOBS, WorkflowJobBean.toJSONArray(jsonWorkflows));
      json.put(JsonTags.WORKFLOWS_TOTAL, jobs.getTotal());
      json.put(JsonTags.WORKFLOWS_OFFSET, jobs.getStart());
      json.put(JsonTags.WORKFLOWS_LEN, jobs.getLen());

    } catch (DagEngineException ex) {
      throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }

    return json;
  }
Exemplo n.º 3
0
  /** v1 service implementation to submit a workflow job */
  @SuppressWarnings("unchecked")
  private JSONObject submitWorkflowJob(HttpServletRequest request, Configuration conf)
      throws XServletException {

    JSONObject json = new JSONObject();

    try {
      String action = request.getParameter(RestConstants.ACTION_PARAM);
      if (action != null && !action.equals(RestConstants.JOB_ACTION_START)) {
        throw new XServletException(
            HttpServletResponse.SC_BAD_REQUEST,
            ErrorCode.E0303,
            RestConstants.ACTION_PARAM,
            action);
      }
      boolean startJob = (action != null);
      String user = conf.get(OozieClient.USER_NAME);
      DagEngine dagEngine =
          Services.get().get(DagEngineService.class).getDagEngine(user, getAuthToken(request));
      String id = dagEngine.submitJob(conf, startJob);
      json.put(JsonTags.JOB_ID, id);
    } catch (DagEngineException ex) {
      throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }

    return json;
  }
Exemplo n.º 4
0
  /**
   * Provides functionality to test for set*Data calls not being made by the Action Handler.
   *
   * @param avoidParam set*Data function call to avoid.
   * @param expActionErrorCode the expected action error code.
   * @throws Exception
   */
  private void _testDataNotSet(String avoidParam, String expActionErrorCode) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("external-status", "ok");
    conf.set("signal-value", "based_on_action_status");
    conf.set(avoidParam, "true");

    final String jobId = engine.submitJob(conf, true);

    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    Thread.sleep(2000);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            WorkflowJobBean bean = store.getWorkflow(jobId, false);
            return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.FAILED);
          }
        });
    store.commitTrx();
    store.closeTrx();

    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    store2.beginTrx();
    assertEquals(
        WorkflowInstance.Status.FAILED,
        store2.getWorkflow(jobId, false).getWorkflowInstance().getStatus());
    assertEquals(WorkflowJob.Status.FAILED, engine.getJob(jobId).getStatus());

    List<WorkflowActionBean> actions = store2.getActionsForWorkflow(jobId, false);
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
      if (bean.getType().equals("test")) {
        action = bean;
        break;
      }
    }
    assertNotNull(action);
    assertEquals(expActionErrorCode, action.getErrorCode());
    store2.commitTrx();
    store2.closeTrx();
  }
Exemplo n.º 5
0
 /**
  * Stream wf job log
  *
  * @param request servlet request
  * @param response servlet response
  * @throws XServletException
  * @throws IOException
  */
 private void streamWorkflowJobLog(HttpServletRequest request, HttpServletResponse response)
     throws XServletException, IOException {
   DagEngine dagEngine =
       Services.get()
           .get(DagEngineService.class)
           .getDagEngine(getUser(request), getAuthToken(request));
   String jobId = getResourceName(request);
   try {
     dagEngine.streamLog(jobId, response.getWriter());
   } catch (DagEngineException ex) {
     throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
   }
 }
Exemplo n.º 6
0
  /**
   * Provides functionality to test user retry
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param externalStatus the external status to set.
   * @param signalValue the signal value to set.
   * @throws Exception
   */
  private void _testErrorWithUserRetry(String errorType, String externalStatus, String signalValue)
      throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid-user-retry.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", errorType);
    conf.set("external-status", externalStatus);
    conf.set("signal-value", signalValue);

    final String jobId = engine.submitJob(conf, true);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);

    final WorkflowActionsGetForJobJPAExecutor actionsGetExecutor =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor);
            WorkflowActionBean action = null;
            for (WorkflowActionBean bean : actions) {
              if (bean.getType().equals("test")) {
                action = bean;
                break;
              }
            }
            return (action != null && action.getUserRetryCount() == 2);
          }
        });

    List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor);
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
      if (bean.getType().equals("test")) {
        action = bean;
        break;
      }
    }
    assertNotNull(action);
    assertEquals(2, action.getUserRetryCount());
  }
Exemplo n.º 7
0
  /**
   * Provides functionality to test kill node message
   *
   * @throws Exception
   */
  public void testKillNodeErrorMessage() throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-test-kill-node-message.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", "end.error");
    conf.set("external-status", "FAILED/KILLED");
    conf.set("signal-value", "fail");

    final String jobId = engine.submitJob(conf, true);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);

    waitFor(
        50000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
            return (job.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
          }
        });

    WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.KILLED, job.getStatus());

    WorkflowActionsGetForJobJPAExecutor wfActionsGetCmd =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actions = jpaService.execute(wfActionsGetCmd);

    int n = actions.size();
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
      if (bean.getType().equals("test")) {
        action = bean;
        break;
      }
    }
    assertNotNull(action);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals("end", action.getErrorMessage());
    assertEquals(WorkflowAction.Status.ERROR, action.getStatus());
  }
Exemplo n.º 8
0
  /**
   * Rerun a wf job
   *
   * @param request servlet request
   * @param response servlet response
   * @param conf configuration object
   * @throws XServletException
   */
  private void reRunWorkflowJob(
      HttpServletRequest request, HttpServletResponse response, Configuration conf)
      throws XServletException {
    DagEngine dagEngine =
        Services.get()
            .get(DagEngineService.class)
            .getDagEngine(getUser(request), getAuthToken(request));

    String jobId = getResourceName(request);
    try {
      dagEngine.reRun(jobId, conf);
    } catch (DagEngineException ex) {
      throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }
  }
Exemplo n.º 9
0
  /** service implementation to submit a http job */
  private JSONObject submitHttpJob(HttpServletRequest request, Configuration conf, String jobType)
      throws XServletException {
    JSONObject json = new JSONObject();

    try {
      String user = conf.get(OozieClient.USER_NAME);
      DagEngine dagEngine =
          Services.get().get(DagEngineService.class).getDagEngine(user, getAuthToken(request));
      String id = dagEngine.submitHttpJob(conf, jobType);
      json.put(JsonTags.JOB_ID, id);
    } catch (DagEngineException ex) {
      throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }

    return json;
  }
Exemplo n.º 10
0
 /** v1 service implementation to get a JSONObject representation of a job from its external ID */
 @SuppressWarnings("unchecked")
 private JSONObject getWorkflowJobIdForExternalId(HttpServletRequest request, String externalId)
     throws XServletException {
   JSONObject json = new JSONObject();
   try {
     DagEngine dagEngine =
         Services.get()
             .get(DagEngineService.class)
             .getDagEngine(getUser(request), getAuthToken(request));
     String jobId = dagEngine.getJobIdForExternalId(externalId);
     json.put(JsonTags.JOB_ID, jobId);
   } catch (DagEngineException ex) {
     throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
   }
   return json;
 }
Exemplo n.º 11
0
  /**
   * Get wf job definition
   *
   * @param request servlet request
   * @param response servlet response
   * @return String wf definition
   * @throws XServletException
   */
  private String getWorkflowJobDefinition(HttpServletRequest request, HttpServletResponse response)
      throws XServletException {
    DagEngine dagEngine =
        Services.get()
            .get(DagEngineService.class)
            .getDagEngine(getUser(request), getAuthToken(request));

    String wfDefinition;
    String jobId = getResourceName(request);
    try {
      wfDefinition = dagEngine.getDefinition(jobId);
    } catch (DagEngineException ex) {
      throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }
    return wfDefinition;
  }
Exemplo n.º 12
0
  /**
   * Get wf action info
   *
   * @param request servlet request
   * @param response servlet response
   * @return JsonBean WorkflowActionBean
   * @throws XServletException
   */
  private JsonBean getWorkflowAction(HttpServletRequest request, HttpServletResponse response)
      throws XServletException {
    DagEngine dagEngine =
        Services.get()
            .get(DagEngineService.class)
            .getDagEngine(getUser(request), getAuthToken(request));

    JsonBean actionBean = null;
    String actionId = getResourceName(request);
    try {
      actionBean = dagEngine.getWorkflowAction(actionId);
    } catch (BaseEngineException ex) {
      throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }

    return actionBean;
  }
Exemplo n.º 13
0
  /**
   * Get workflow job
   *
   * @param request servlet request
   * @param response servlet response
   * @return JsonBean WorkflowJobBean
   * @throws XServletException
   */
  private JsonBean getWorkflowJob(HttpServletRequest request, HttpServletResponse response)
      throws XServletException {
    JsonBean jobBean = null;
    String jobId = getResourceName(request);
    String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
    String lenStr = request.getParameter(RestConstants.LEN_PARAM);
    int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
    start = (start < 1) ? 1 : start;
    int len = (lenStr != null) ? Integer.parseInt(lenStr) : 0;
    len = (len < 1) ? Integer.MAX_VALUE : len;
    DagEngine dagEngine =
        Services.get()
            .get(DagEngineService.class)
            .getDagEngine(getUser(request), getAuthToken(request));
    try {
      jobBean = (JsonBean) dagEngine.getJob(jobId, start, len);
    } catch (DagEngineException ex) {
      throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }

    return jobBean;
  }
Exemplo n.º 14
0
  /**
   * Provides functionality to test transient failures.
   *
   * @param errorType the error type. (start.transient, end.transient)
   * @param expStatus1 expected status after the first step (START_RETRY, END_RETRY)
   * @param expStatus2 expected status after the second step (START_MANUAL, END_MANUAL)
   * @param expErrorMsg the expected error message.
   * @throws Exception
   */
  private void _testTransient(
      String errorType,
      WorkflowActionBean.Status expStatus1,
      final WorkflowActionBean.Status expStatus2,
      String expErrorMsg)
      throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final int maxRetries = 2;
    final int retryInterval = 10;

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);
    conf.setInt(OozieClient.ACTION_MAX_RETRIES, maxRetries);
    conf.setInt(OozieClient.ACTION_RETRY_INTERVAL, retryInterval);

    final String jobId = engine.submitJob(conf, true);

    int retryCount = 1;
    WorkflowActionBean.Status expectedStatus = expStatus1;
    int expectedRetryCount = 2;

    Thread.sleep(20000);
    String aId = null;
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    while (retryCount <= maxRetries) {
      List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false);
      WorkflowActionBean action = null;
      for (WorkflowActionBean bean : actions) {
        if (bean.getType().equals("test")) {
          action = bean;
          break;
        }
      }
      assertNotNull(action);
      aId = action.getId();
      assertEquals(expectedStatus, action.getStatus());
      assertEquals(expectedRetryCount, action.getRetries());
      assertEquals("TEST_ERROR", action.getErrorCode());
      assertEquals(expErrorMsg, action.getErrorMessage());
      if (action.getRetries() == maxRetries) {
        expectedRetryCount = 0;
        expectedStatus = expStatus2;
        break;
      } else {
        expectedRetryCount++;
      }
      Thread.sleep(retryInterval * 1000);
      retryCount++;
    }
    store.commitTrx();
    store.closeTrx();
    Thread.sleep(5000);

    final String actionId = aId;

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getWorkflowAction(actionId).getStatus() == expStatus2);
          }
        });

    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    store2.beginTrx();
    WorkflowActionBean action = engine.getWorkflowAction(actionId);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus2, action.getStatus());
    assertTrue(action.isPending() == false);
    assertEquals(WorkflowJob.Status.SUSPENDED, engine.getJob(jobId).getStatus());
    store2.commitTrx();
    store2.closeTrx();
  }
Exemplo n.º 15
0
  /**
   * Provides functionality to test non transient failures and coordinator action update
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
   * @param expErrorMsg expected error message.
   * @throws Exception
   */
  private void _testNonTransientWithCoordActionUpdate(
      String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);

    final String jobId = engine.submitJob(conf, false);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final CoordinatorJobBean coordJob =
        addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, false, false);
    CoordinatorActionBean coordAction =
        addRecordToCoordActionTable(
            coordJob.getId(),
            1,
            CoordinatorAction.Status.RUNNING,
            "coord-action-get.xml",
            jobId,
            "RUNNING",
            0);

    engine.start(jobId);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
          }
        });

    assertNotNull(jpaService);
    WorkflowJobGetJPAExecutor wfGetCmd = new WorkflowJobGetJPAExecutor(jobId);
    WorkflowJobBean job = jpaService.execute(wfGetCmd);
    WorkflowActionsGetForJobJPAExecutor actionsGetExe =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actionsList = jpaService.execute(actionsGetExe);

    int n = actionsList.size();
    WorkflowActionBean action = actionsList.get(n - 1);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus1, action.getStatus());
    assertFalse(action.isPending());

    assertEquals(WorkflowJob.Status.SUSPENDED, job.getStatus());

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            CoordinatorActionBean coordAction2 =
                jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId));
            return coordAction2.getStatus().equals(CoordinatorAction.Status.SUSPENDED);
          }
        });

    coordAction = jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId));
    assertEquals(CoordinatorAction.Status.SUSPENDED, coordAction.getStatus());
  }
Exemplo n.º 16
0
  /**
   * Provides functionality to test non transient failures.
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
   * @param expErrorMsg expected error message.
   * @throws Exception
   */
  private void _testNonTransient(
      String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);

    final String jobId = engine.submitJob(conf, true);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
          }
        });

    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, true);
    int n = actions.size();
    WorkflowActionBean action = actions.get(n - 1);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus1, action.getStatus());
    assertTrue(action.isPending() == false);

    assertTrue(engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);

    String actionConf = action.getConf();
    String fixedActionConf = actionConf.replaceAll(errorType, "none");
    action.setConf(fixedActionConf);
    store.updateAction(action);
    store.commitTrx();
    store.closeTrx();

    engine.resume(jobId);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED);
          }
        });

    assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());

    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    store2.beginTrx();
    actions = store2.getActionsForWorkflow(jobId, false);
    action = actions.get(0);
    assertEquals(null, action.getErrorCode());
    assertEquals(null, action.getErrorMessage());
    assertEquals(WorkflowActionBean.Status.OK, action.getStatus());
    store2.commitTrx();
    store2.closeTrx();
  }