/** * Provides functionality to test for set*Data calls not being made by the Action Handler. * * @param avoidParam set*Data function call to avoid. * @param expActionErrorCode the expected action error code. * @throws Exception */ private void _testDataNotSet(String avoidParam, String expActionErrorCode) throws Exception { String workflowPath = getTestCaseFileUri("workflow.xml"); Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml")); IOUtils.copyCharStream(reader, writer); final DagEngine engine = new DagEngine("u"); Configuration conf = new XConfiguration(); conf.set(OozieClient.APP_PATH, workflowPath); conf.set(OozieClient.USER_NAME, getTestUser()); conf.set(OozieClient.LOG_TOKEN, "t"); conf.set("external-status", "ok"); conf.set("signal-value", "based_on_action_status"); conf.set(avoidParam, "true"); final String jobId = engine.submitJob(conf, true); final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); store.beginTrx(); Thread.sleep(2000); waitFor( 5000, new Predicate() { public boolean evaluate() throws Exception { WorkflowJobBean bean = store.getWorkflow(jobId, false); return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.FAILED); } }); store.commitTrx(); store.closeTrx(); final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create(); store2.beginTrx(); assertEquals( WorkflowInstance.Status.FAILED, store2.getWorkflow(jobId, false).getWorkflowInstance().getStatus()); assertEquals(WorkflowJob.Status.FAILED, engine.getJob(jobId).getStatus()); List<WorkflowActionBean> actions = store2.getActionsForWorkflow(jobId, false); WorkflowActionBean action = null; for (WorkflowActionBean bean : actions) { if (bean.getType().equals("test")) { action = bean; break; } } assertNotNull(action); assertEquals(expActionErrorCode, action.getErrorCode()); store2.commitTrx(); store2.closeTrx(); }
/** * Provides functionality to test user retry * * @param errorType the error type. (start.non-transient, end.non-transient) * @param externalStatus the external status to set. * @param signalValue the signal value to set. * @throws Exception */ private void _testErrorWithUserRetry(String errorType, String externalStatus, String signalValue) throws Exception { String workflowPath = getTestCaseFileUri("workflow.xml"); Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid-user-retry.xml", -1); Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml")); IOUtils.copyCharStream(reader, writer); final DagEngine engine = new DagEngine("u"); Configuration conf = new XConfiguration(); conf.set(OozieClient.APP_PATH, workflowPath); conf.set(OozieClient.USER_NAME, getTestUser()); conf.set(OozieClient.LOG_TOKEN, "t"); conf.set("error", errorType); conf.set("external-status", externalStatus); conf.set("signal-value", signalValue); final String jobId = engine.submitJob(conf, true); final JPAService jpaService = Services.get().get(JPAService.class); final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId); final WorkflowActionsGetForJobJPAExecutor actionsGetExecutor = new WorkflowActionsGetForJobJPAExecutor(jobId); waitFor( 5000, new Predicate() { public boolean evaluate() throws Exception { List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor); WorkflowActionBean action = null; for (WorkflowActionBean bean : actions) { if (bean.getType().equals("test")) { action = bean; break; } } return (action != null && action.getUserRetryCount() == 2); } }); List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor); WorkflowActionBean action = null; for (WorkflowActionBean bean : actions) { if (bean.getType().equals("test")) { action = bean; break; } } assertNotNull(action); assertEquals(2, action.getUserRetryCount()); }
/** * Provides functionality to test kill node message * * @throws Exception */ public void testKillNodeErrorMessage() throws Exception { String workflowPath = getTestCaseFileUri("workflow.xml"); Reader reader = IOUtils.getResourceAsReader("wf-test-kill-node-message.xml", -1); Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml")); IOUtils.copyCharStream(reader, writer); final DagEngine engine = new DagEngine("u"); Configuration conf = new XConfiguration(); conf.set(OozieClient.APP_PATH, workflowPath); conf.set(OozieClient.USER_NAME, getTestUser()); conf.set(OozieClient.LOG_TOKEN, "t"); conf.set("error", "end.error"); conf.set("external-status", "FAILED/KILLED"); conf.set("signal-value", "fail"); final String jobId = engine.submitJob(conf, true); final JPAService jpaService = Services.get().get(JPAService.class); final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId); waitFor( 50000, new Predicate() { public boolean evaluate() throws Exception { WorkflowJobBean job = jpaService.execute(wfJobGetCmd); return (job.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED); } }); WorkflowJobBean job = jpaService.execute(wfJobGetCmd); assertEquals(WorkflowJob.Status.KILLED, job.getStatus()); WorkflowActionsGetForJobJPAExecutor wfActionsGetCmd = new WorkflowActionsGetForJobJPAExecutor(jobId); List<WorkflowActionBean> actions = jpaService.execute(wfActionsGetCmd); int n = actions.size(); WorkflowActionBean action = null; for (WorkflowActionBean bean : actions) { if (bean.getType().equals("test")) { action = bean; break; } } assertNotNull(action); assertEquals("TEST_ERROR", action.getErrorCode()); assertEquals("end", action.getErrorMessage()); assertEquals(WorkflowAction.Status.ERROR, action.getStatus()); }
protected Void call(WorkflowStore store) throws StoreException, CommandException { WorkflowJobBean workflow = store.getWorkflow(jobId, false); setLogInfo(workflow); WorkflowActionBean action = store.getAction(id, false); setLogInfo(action); if (action.isPending() && (action.getStatus() == WorkflowActionBean.Status.DONE || action.getStatus() == WorkflowActionBean.Status.END_RETRY || action.getStatus() == WorkflowActionBean.Status.END_MANUAL)) { if (workflow.getStatus() == WorkflowJob.Status.RUNNING) { ActionExecutor executor = Services.get().get(ActionService.class).getExecutor(action.getType()); Configuration conf = workflow.getWorkflowInstance().getConf(); int maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries()); long retryInterval = conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval()); executor.setMaxRetries(maxRetries); executor.setRetryInterval(retryInterval); if (executor != null) { boolean isRetry = false; if (action.getStatus() == WorkflowActionBean.Status.END_RETRY || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) { isRetry = true; } ActionExecutorContext context = new ActionCommand.ActionExecutorContext(workflow, action, isRetry); try { XLog.getLog(getClass()) .debug( "End, name [{0}] type [{1}] status[{2}] external status [{3}] signal value [{4}]", action.getName(), action.getType(), action.getStatus(), action.getExternalStatus(), action.getSignalValue()); WorkflowInstance wfInstance = workflow.getWorkflowInstance(); DagELFunctions.setActionInfo(wfInstance, action); workflow.setWorkflowInstance(wfInstance); incrActionCounter(action.getType(), 1); Instrumentation.Cron cron = new Instrumentation.Cron(); cron.start(); executor.end(context, action); cron.stop(); addActionCron(action.getType(), cron); if (!context.isEnded()) { XLog.getLog(getClass()) .warn( XLog.OPS, "Action Ended, ActionExecutor [{0}] must call setEndData()", executor.getType()); action.setErrorInfo( END_DATA_MISSING, "Execution Ended, but End Data Missing from Action"); failJob(context); store.updateAction(action); store.updateWorkflow(workflow); return null; } action.setRetries(0); action.setEndTime(new Date()); store.updateAction(action); store.updateWorkflow(workflow); Status slaStatus = null; switch (action.getStatus()) { case OK: slaStatus = Status.SUCCEEDED; break; case KILLED: slaStatus = Status.KILLED; break; case FAILED: slaStatus = Status.FAILED; break; case ERROR: XLog.getLog(getClass()).info("ERROR is considered as FAILED for SLA"); slaStatus = Status.KILLED; break; default: // TODO: What will happen for other Action // status slaStatus = Status.FAILED; break; } SLADbOperations.writeStausEvent( action.getSlaXml(), action.getId(), store, slaStatus, SlaAppType.WORKFLOW_ACTION); queueCallable(new NotificationCommand(workflow, action)); XLog.getLog(getClass()) .debug( "Queuing commands for action " + id + " status " + action.getStatus() + ", Set pending=" + action.getPending()); queueCallable(new SignalCommand(workflow.getId(), id)); } catch (ActionExecutorException ex) { XLog.getLog(getClass()) .warn( "Error ending action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]", action.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage()); action.setErrorInfo(ex.getErrorCode(), ex.getMessage()); action.setEndTime(null); switch (ex.getErrorType()) { case TRANSIENT: if (!handleTransient(context, executor, WorkflowAction.Status.END_RETRY)) { handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL); action.setPendingAge(new Date()); action.setRetries(0); } action.setEndTime(null); break; case NON_TRANSIENT: handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL); action.setEndTime(null); break; case ERROR: handleError(context, executor, COULD_NOT_END, false, WorkflowAction.Status.ERROR); queueCallable(new SignalCommand(workflow.getId(), id)); break; case FAILED: failJob(context); break; } store.updateAction(action); store.updateWorkflow(workflow); } } else { throw new CommandException(ErrorCode.E0802, action.getType()); } } else { XLog.getLog(getClass()) .warn( "Job state is not {0}. Skipping ActionEnd Execution", WorkflowJob.Status.RUNNING.toString()); } } else { XLog.getLog(getClass()) .debug( "Action pending={0}, status={1}. Skipping ActionEnd Execution", action.getPending(), action.getStatusStr()); } return null; }
public void testGetUpdateQuery() throws Exception { EntityManager em = jpaService.getEntityManager(); WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); WorkflowActionBean bean = addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP); // UPDATE_ACTION Query query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em); assertEquals(query.getParameterValue("conf"), bean.getConfBlob()); assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl()); assertEquals(query.getParameterValue("data"), bean.getDataBlob()); assertEquals(query.getParameterValue("stats"), bean.getStatsBlob()); assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob()); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("externalId"), bean.getExternalId()); assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus()); assertEquals(query.getParameterValue("name"), bean.getName()); assertEquals(query.getParameterValue("cred"), bean.getCred()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri()); assertEquals(query.getParameterValue("transition"), bean.getTransition()); assertEquals(query.getParameterValue("type"), bean.getType()); assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp()); assertEquals(query.getParameterValue("executionPath"), bean.getExecutionPath()); assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp()); assertEquals(query.getParameterValue("logToken"), bean.getLogToken()); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAge()); assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue()); assertEquals(query.getParameterValue("slaXml"), bean.getSlaXmlBlob()); assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("wfId"), bean.getWfId()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_FOR_LAST_CHECKED_TIME query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em); assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_PENDING query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING, bean, em); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_STATUS_PENDING query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, bean, em); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_PENDING_TRANS query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS, bean, em); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("transition"), bean.getTransition()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_PENDING_TRANS_ERROR query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS_ERROR, bean, em); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("transition"), bean.getTransition()); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_START query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_START, bean, em); assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp()); assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob()); assertEquals(query.getParameterValue("conf"), bean.getConfBlob()); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("externalId"), bean.getExternalId()); assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri()); assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl()); assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus()); assertEquals(query.getParameterValue("data"), bean.getDataBlob()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_CHECK query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_CHECK, bean, em); assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob()); assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("data"), bean.getDataBlob()); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp()); assertEquals(query.getParameterValue("stats"), bean.getStatsBlob()); assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_END query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_END, bean, em); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue()); assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount()); assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus()); assertEquals(query.getParameterValue("stats"), bean.getStatsBlob()); assertEquals(query.getParameterValue("id"), bean.getId()); em.close(); }
public void testGet() throws Exception { WorkflowActionBean bean = addRecordToWfActionTable("workflowId", "testAction", WorkflowAction.Status.PREP, "", true); WorkflowActionBean retBean; // GET_ACTION_ID_TYPE_LASTCHECK retBean = WorkflowActionQueryExecutor.getInstance() .get(WorkflowActionQuery.GET_ACTION_ID_TYPE_LASTCHECK, bean.getId()); assertEquals(bean.getId(), retBean.getId()); assertEquals(bean.getType(), retBean.getType()); assertEquals(bean.getLastCheckTime(), retBean.getLastCheckTime()); // GET_ACTION_FAIL retBean = WorkflowActionQueryExecutor.getInstance() .get(WorkflowActionQuery.GET_ACTION_FAIL, bean.getId()); assertEquals(bean.getId(), retBean.getId()); assertEquals(bean.getJobId(), retBean.getJobId()); assertEquals(bean.getName(), retBean.getName()); assertEquals(bean.getStatusStr(), retBean.getStatusStr()); assertEquals(bean.getPending(), retBean.getPending()); assertEquals(bean.getType(), retBean.getType()); assertEquals(bean.getLogToken(), retBean.getLogToken()); assertEquals(bean.getTransition(), retBean.getTransition()); assertEquals(bean.getErrorCode(), retBean.getErrorCode()); assertEquals(bean.getErrorMessage(), retBean.getErrorMessage()); assertNull(retBean.getConf()); assertNull(retBean.getSlaXml()); assertNull(retBean.getData()); assertNull(retBean.getStats()); assertNull(retBean.getExternalChildIDs()); // GET_ACTION_SIGNAL retBean = WorkflowActionQueryExecutor.getInstance() .get(WorkflowActionQuery.GET_ACTION_SIGNAL, bean.getId()); assertEquals(bean.getId(), retBean.getId()); assertEquals(bean.getJobId(), retBean.getJobId()); assertEquals(bean.getName(), retBean.getName()); assertEquals(bean.getStatusStr(), retBean.getStatusStr()); assertEquals(bean.getPending(), retBean.getPending()); assertEquals(bean.getPendingAge().getTime(), retBean.getPendingAge().getTime()); assertEquals(bean.getType(), retBean.getType()); assertEquals(bean.getLogToken(), retBean.getLogToken()); assertEquals(bean.getTransition(), retBean.getTransition()); assertEquals(bean.getErrorCode(), retBean.getErrorCode()); assertEquals(bean.getErrorMessage(), retBean.getErrorMessage()); assertEquals(bean.getExecutionPath(), retBean.getExecutionPath()); assertEquals(bean.getSignalValue(), retBean.getSignalValue()); assertEquals(bean.getSlaXml(), retBean.getSlaXml()); assertNull(retBean.getConf()); assertNull(retBean.getData()); assertNull(retBean.getStats()); assertNull(retBean.getExternalChildIDs()); // GET_ACTION_CHECK retBean = WorkflowActionQueryExecutor.getInstance() .get(WorkflowActionQuery.GET_ACTION_CHECK, bean.getId()); assertEquals(bean.getId(), retBean.getId()); assertEquals(bean.getJobId(), retBean.getJobId()); assertEquals(bean.getName(), retBean.getName()); assertEquals(bean.getStatusStr(), retBean.getStatusStr()); assertEquals(bean.getPending(), retBean.getPending()); assertEquals(bean.getPendingAge().getTime(), retBean.getPendingAge().getTime()); assertEquals(bean.getType(), retBean.getType()); assertEquals(bean.getLogToken(), retBean.getLogToken()); assertEquals(bean.getTransition(), retBean.getTransition()); assertEquals(bean.getRetries(), retBean.getRetries()); assertEquals(bean.getUserRetryCount(), retBean.getUserRetryCount()); assertEquals(bean.getUserRetryMax(), retBean.getUserRetryMax()); assertEquals(bean.getUserRetryInterval(), retBean.getUserRetryInterval()); assertEquals(bean.getTrackerUri(), retBean.getTrackerUri()); assertEquals(bean.getStartTime().getTime(), retBean.getStartTime().getTime()); assertEquals(bean.getEndTime().getTime(), retBean.getEndTime().getTime()); assertEquals(bean.getLastCheckTime().getTime(), retBean.getLastCheckTime().getTime()); assertEquals(bean.getErrorCode(), retBean.getErrorCode()); assertEquals(bean.getErrorMessage(), retBean.getErrorMessage()); assertEquals(bean.getExternalId(), retBean.getExternalId()); assertEquals(bean.getExternalStatus(), retBean.getExternalStatus()); assertEquals(bean.getExternalChildIDs(), retBean.getExternalChildIDs()); assertEquals(bean.getConf(), retBean.getConf()); assertNull(retBean.getData()); assertNull(retBean.getStats()); assertNull(retBean.getSlaXml()); // GET_ACTION_END retBean = WorkflowActionQueryExecutor.getInstance() .get(WorkflowActionQuery.GET_ACTION_END, bean.getId()); assertEquals(bean.getId(), retBean.getId()); assertEquals(bean.getJobId(), retBean.getJobId()); assertEquals(bean.getName(), retBean.getName()); assertEquals(bean.getStatusStr(), retBean.getStatusStr()); assertEquals(bean.getPending(), retBean.getPending()); assertEquals(bean.getPendingAge().getTime(), retBean.getPendingAge().getTime()); assertEquals(bean.getType(), retBean.getType()); assertEquals(bean.getLogToken(), retBean.getLogToken()); assertEquals(bean.getTransition(), retBean.getTransition()); assertEquals(bean.getRetries(), retBean.getRetries()); assertEquals(bean.getTrackerUri(), retBean.getTrackerUri()); assertEquals(bean.getUserRetryCount(), retBean.getUserRetryCount()); assertEquals(bean.getUserRetryMax(), retBean.getUserRetryMax()); assertEquals(bean.getUserRetryInterval(), retBean.getUserRetryInterval()); assertEquals(bean.getExternalId(), retBean.getExternalId()); assertEquals(bean.getExternalStatus(), retBean.getExternalStatus()); assertEquals(bean.getExternalChildIDs(), retBean.getExternalChildIDs()); assertEquals(bean.getStartTime().getTime(), retBean.getStartTime().getTime()); assertEquals(bean.getEndTime().getTime(), retBean.getEndTime().getTime()); assertEquals(bean.getErrorCode(), retBean.getErrorCode()); assertEquals(bean.getErrorMessage(), retBean.getErrorMessage()); assertEquals(bean.getConf(), retBean.getConf()); assertEquals(bean.getData(), retBean.getData()); assertEquals(bean.getStats(), retBean.getStats()); assertNull(retBean.getSlaXml()); // GET_ACTION_COMPLETED retBean = WorkflowActionQueryExecutor.getInstance() .get(WorkflowActionQuery.GET_ACTION_COMPLETED, bean.getId()); assertEquals(bean.getId(), retBean.getId()); assertEquals(bean.getJobId(), retBean.getJobId()); assertEquals(bean.getStatusStr(), retBean.getStatusStr()); assertEquals(bean.getType(), retBean.getType()); assertEquals(bean.getLogToken(), retBean.getLogToken()); assertNull(retBean.getSlaXml()); assertNull(retBean.getConf()); assertNull(retBean.getData()); assertNull(retBean.getStats()); assertNull(retBean.getExternalChildIDs()); // GET_ACTION (entire obj) retBean = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION, bean.getId()); assertEquals(bean.getId(), retBean.getId()); assertEquals(bean.getJobId(), retBean.getJobId()); assertEquals(bean.getName(), retBean.getName()); assertEquals(bean.getStatusStr(), retBean.getStatusStr()); assertEquals(bean.getPending(), retBean.getPending()); assertEquals(bean.getPendingAge().getTime(), retBean.getPendingAge().getTime()); assertEquals(bean.getType(), retBean.getType()); assertEquals(bean.getLogToken(), retBean.getLogToken()); assertEquals(bean.getTransition(), retBean.getTransition()); assertEquals(bean.getRetries(), retBean.getRetries()); assertEquals(bean.getUserRetryCount(), retBean.getUserRetryCount()); assertEquals(bean.getUserRetryMax(), retBean.getUserRetryMax()); assertEquals(bean.getUserRetryInterval(), retBean.getUserRetryInterval()); assertEquals(bean.getStartTime().getTime(), retBean.getStartTime().getTime()); assertEquals(bean.getEndTime().getTime(), retBean.getEndTime().getTime()); assertEquals(bean.getCreatedTime().getTime(), retBean.getCreatedTime().getTime()); assertEquals(bean.getLastCheckTime().getTime(), retBean.getLastCheckTime().getTime()); assertEquals(bean.getErrorCode(), retBean.getErrorCode()); assertEquals(bean.getErrorMessage(), retBean.getErrorMessage()); assertEquals(bean.getExecutionPath(), retBean.getExecutionPath()); assertEquals(bean.getSignalValue(), retBean.getSignalValue()); assertEquals(bean.getCred(), retBean.getCred()); assertEquals(bean.getConf(), retBean.getConf()); assertEquals(bean.getSlaXml(), retBean.getSlaXml()); assertEquals(bean.getData(), retBean.getData()); assertEquals(bean.getStats(), retBean.getStats()); assertEquals(bean.getExternalChildIDs(), retBean.getExternalChildIDs()); }
public void testFsFailover() throws Exception { Reader reader = IOUtils.getResourceAsReader("failover-fs-wf.xml", -1); Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); IOUtils.copyCharStream(reader, writer); final OozieClient wfClient = LocalOozie.getClient(); Properties conf = wfClient.createConfiguration(); conf.setProperty(OozieClient.APP_PATH, getTestCaseDir()); conf.setProperty(OozieClient.USER_NAME, getTestUser()); conf.setProperty(OozieClient.GROUP_NAME, getTestGroup()); injectKerberosInfo(conf); final Path source = new Path(getFsTestCaseDir(), "fsfailover-source"); getFileSystem().mkdirs(source); final Path target = new Path(getFsTestCaseDir().toString(), "fsfailover-target"); conf.setProperty("source", source.toString()); conf.setProperty("target", target.toUri().getPath()); setSystemProperty(FaultInjection.FAULT_INJECTION, "true"); setSystemProperty(SkipCommitFaultInjection.ACTION_FAILOVER_FAULT_INJECTION, "true"); final String jobId1 = wfClient.submit(conf); wfClient.start(jobId1); waitFor( 10 * 1000, new Predicate() { public boolean evaluate() throws Exception { return getFileSystem().exists(target); } }); assertTrue(getFileSystem().exists(target)); waitFor( 10 * 1000, new Predicate() { public boolean evaluate() throws Exception { return FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection"); } }); assertFalse(FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection")); assertEquals(WorkflowJob.Status.RUNNING, wfClient.getJobInfo(jobId1).getStatus()); WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId1, false); WorkflowActionBean action = actions.get(0); assertEquals(WorkflowAction.Status.PREP, action.getStatus()); setSystemProperty(FaultInjection.FAULT_INJECTION, "false"); setSystemProperty(SkipCommitFaultInjection.ACTION_FAILOVER_FAULT_INJECTION, "false"); ActionStartCommand actionStartCommand = new ActionStartCommand(action.getId(), action.getType()); actionStartCommand.call(); store = Services.get().get(WorkflowStoreService.class).create(); actions = store.getActionsForWorkflow(jobId1, false); action = actions.get(0); assertEquals(WorkflowAction.Status.DONE, action.getStatus()); waitFor( 5 * 1000, new Predicate() { public boolean evaluate() throws Exception { return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.SUCCEEDED; } }); assertEquals(WorkflowJob.Status.SUCCEEDED, wfClient.getJobInfo(jobId1).getStatus()); final String jobId2 = wfClient.submit(conf); wfClient.start(jobId2); waitFor( 10 * 1000, new Predicate() { public boolean evaluate() throws Exception { return wfClient.getJobInfo(jobId2).getStatus() == WorkflowJob.Status.KILLED; } }); assertEquals(WorkflowJob.Status.KILLED, wfClient.getJobInfo(jobId2).getStatus()); }
@Override protected Void execute() throws CommandException { LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId); WorkflowInstance workflowInstance = wfJob.getWorkflowInstance(); workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob); boolean completed = false; boolean skipAction = false; if (wfAction == null) { if (wfJob.getStatus() == WorkflowJob.Status.PREP) { try { completed = workflowInstance.start(); } catch (WorkflowException e) { throw new CommandException(e); } wfJob.setStatus(WorkflowJob.Status.RUNNING); wfJob.setStartTime(new Date()); wfJob.setWorkflowInstance(workflowInstance); // 1. Add SLA status event for WF-JOB with status STARTED // 2. Add SLA registration events for all WF_ACTIONS SLADbXOperations.writeStausEvent( wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB); writeSLARegistrationForAllActions( workflowInstance.getApp().getDefinition(), wfJob.getUser(), wfJob.getGroup(), wfJob.getConf()); queue(new NotificationXCommand(wfJob)); } else { throw new CommandException(ErrorCode.E0801, wfJob.getId()); } } else { String skipVar = workflowInstance.getVar( wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP); if (skipVar != null) { skipAction = skipVar.equals("true"); } try { completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue()); } catch (WorkflowException e) { throw new CommandException(e); } wfJob.setWorkflowInstance(workflowInstance); wfAction.resetPending(); if (!skipAction) { wfAction.setTransition(workflowInstance.getTransition(wfAction.getName())); } try { jpaService.execute(new WorkflowActionUpdateJPAExecutor(wfAction)); } catch (JPAExecutorException je) { throw new CommandException(je); } } if (completed) { try { for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) { WorkflowActionBean actionToKill; actionToKill = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToKillId)); actionToKill.setPending(); actionToKill.setStatus(WorkflowActionBean.Status.KILLED); jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToKill)); queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType())); } for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) { WorkflowActionBean actionToFail = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToFailId)); actionToFail.resetPending(); actionToFail.setStatus(WorkflowActionBean.Status.FAILED); SLADbXOperations.writeStausEvent( wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION); jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToFail)); } } catch (JPAExecutorException je) { throw new CommandException(je); } wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString())); wfJob.setEndTime(new Date()); wfJob.setWorkflowInstance(workflowInstance); Status slaStatus = Status.SUCCEEDED; switch (wfJob.getStatus()) { case SUCCEEDED: slaStatus = Status.SUCCEEDED; break; case KILLED: slaStatus = Status.KILLED; break; case FAILED: slaStatus = Status.FAILED; break; default: // TODO SUSPENDED break; } SLADbXOperations.writeStausEvent( wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB); queue(new NotificationXCommand(wfJob)); if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) { InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation()); } } else { for (WorkflowActionBean newAction : WorkflowStoreService.getStartedActions(workflowInstance)) { String skipVar = workflowInstance.getVar( newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP); boolean skipNewAction = false; if (skipVar != null) { skipNewAction = skipVar.equals("true"); } try { if (skipNewAction) { WorkflowActionBean oldAction; oldAction = jpaService.execute(new WorkflowActionGetJPAExecutor(newAction.getId())); oldAction.setPending(); jpaService.execute(new WorkflowActionUpdateJPAExecutor(oldAction)); queue(new SignalXCommand(jobId, oldAction.getId())); } else { newAction.setPending(); String actionSlaXml = getActionSLAXml( newAction.getName(), workflowInstance.getApp().getDefinition(), wfJob.getConf()); newAction.setSlaXml(actionSlaXml); jpaService.execute(new WorkflowActionInsertJPAExecutor(newAction)); LOG.debug( "SignalXCommand: Name: " + newAction.getName() + ", Id: " + newAction.getId() + ", Authcode:" + newAction.getCred()); queue(new ActionStartXCommand(newAction.getId(), newAction.getType())); } } catch (JPAExecutorException je) { throw new CommandException(je); } } } try { jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob)); } catch (JPAExecutorException je) { throw new CommandException(je); } XLog.getLog(getClass()) .debug( "Updated the workflow status to " + wfJob.getId() + " status =" + wfJob.getStatusStr()); if (wfJob.getStatus() != WorkflowJob.Status.RUNNING && wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) { // update coordinator action new CoordActionUpdateXCommand(wfJob).call(); new WfEndXCommand(wfJob).call(); // To delete the WF temp dir } LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId); return null; }
/** * Provides functionality to test transient failures. * * @param errorType the error type. (start.transient, end.transient) * @param expStatus1 expected status after the first step (START_RETRY, END_RETRY) * @param expStatus2 expected status after the second step (START_MANUAL, END_MANUAL) * @param expErrorMsg the expected error message. * @throws Exception */ private void _testTransient( String errorType, WorkflowActionBean.Status expStatus1, final WorkflowActionBean.Status expStatus2, String expErrorMsg) throws Exception { String workflowPath = getTestCaseFileUri("workflow.xml"); Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml")); IOUtils.copyCharStream(reader, writer); final int maxRetries = 2; final int retryInterval = 10; final DagEngine engine = new DagEngine("u"); Configuration conf = new XConfiguration(); conf.set(OozieClient.APP_PATH, workflowPath); conf.set(OozieClient.USER_NAME, getTestUser()); conf.set(OozieClient.LOG_TOKEN, "t"); conf.set("signal-value", "OK"); conf.set("external-status", "ok"); conf.set("error", errorType); conf.setInt(OozieClient.ACTION_MAX_RETRIES, maxRetries); conf.setInt(OozieClient.ACTION_RETRY_INTERVAL, retryInterval); final String jobId = engine.submitJob(conf, true); int retryCount = 1; WorkflowActionBean.Status expectedStatus = expStatus1; int expectedRetryCount = 2; Thread.sleep(20000); String aId = null; final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); store.beginTrx(); while (retryCount <= maxRetries) { List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false); WorkflowActionBean action = null; for (WorkflowActionBean bean : actions) { if (bean.getType().equals("test")) { action = bean; break; } } assertNotNull(action); aId = action.getId(); assertEquals(expectedStatus, action.getStatus()); assertEquals(expectedRetryCount, action.getRetries()); assertEquals("TEST_ERROR", action.getErrorCode()); assertEquals(expErrorMsg, action.getErrorMessage()); if (action.getRetries() == maxRetries) { expectedRetryCount = 0; expectedStatus = expStatus2; break; } else { expectedRetryCount++; } Thread.sleep(retryInterval * 1000); retryCount++; } store.commitTrx(); store.closeTrx(); Thread.sleep(5000); final String actionId = aId; waitFor( 5000, new Predicate() { public boolean evaluate() throws Exception { return (engine.getWorkflowAction(actionId).getStatus() == expStatus2); } }); final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create(); store2.beginTrx(); WorkflowActionBean action = engine.getWorkflowAction(actionId); assertEquals("TEST_ERROR", action.getErrorCode()); assertEquals(expErrorMsg, action.getErrorMessage()); assertEquals(expStatus2, action.getStatus()); assertTrue(action.isPending() == false); assertEquals(WorkflowJob.Status.SUSPENDED, engine.getJob(jobId).getStatus()); store2.commitTrx(); store2.closeTrx(); }