public void testWorkflowActionUpdate() throws Exception { WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); WorkflowActionBean action = addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP); _testUpdateAction(action.getId()); }
public void testConfigNotPropagation() throws Exception { Path subWorkflowAppPath = getFsTestCaseDir(); FileSystem fs = getFileSystem(); Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); writer.write(APP1); writer.close(); XConfiguration protoConf = getBaseProtoConf(); WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); String defaultConf = workflow.getConf(); XConfiguration newConf = new XConfiguration(new StringReader(defaultConf)); newConf.set("abc", "xyz"); workflow.setConf(newConf.toXmlString()); final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>" + " <app-path>" + subWorkflowAppPath + File.separator + "workflow.xml" + "</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"); SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); subWorkflow.start(new Context(workflow, action), action); final OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); waitFor( JOB_TIMEOUT, new Predicate() { public boolean evaluate() throws Exception { return oozieClient.getJobInfo(action.getExternalId()).getStatus() == WorkflowJob.Status.SUCCEEDED; } }); assertEquals( WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus()); subWorkflow.check(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.DONE, action.getStatus()); subWorkflow.end(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.OK, action.getStatus()); WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId()); Configuration childConf = new XConfiguration(new StringReader(wf.getConf())); assertNull(childConf.get("abc")); }
/** * Test : verify the PreconditionException is thrown when actionCheckDelay > 0 * * @throws Exception */ public void testActionCheckPreCondition1() throws Exception { Instrumentation inst = Services.get().get(InstrumentationService.class).get(); WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); WorkflowActionBean action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP); ActionCheckXCommand checkCmd = new ActionCheckXCommand(action.getId(), 10); long counterVal; try { counterVal = inst.getCounters() .get(XCommand.INSTRUMENTATION_GROUP) .get(checkCmd.getName() + ".preconditionfailed") .getValue(); } catch (NullPointerException e) { // counter might be null counterVal = 0L; } assertEquals(0L, counterVal); checkCmd.call(); // precondition failed because of actionCheckDelay > 0 counterVal = inst.getCounters() .get(XCommand.INSTRUMENTATION_GROUP) .get(checkCmd.getName() + ".preconditionfailed") .getValue(); assertEquals(1L, counterVal); }
public void testSubWorkflowConfCreation() throws Exception { SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); XConfiguration protoConf = getBaseProtoConf(); WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1'>" + " <app-path>hdfs://foo:9000/user/bar/workflow.xml</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"); OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); assertNotNull(oozieClient); oozieClient = subWorkflow.getWorkflowClient(new Context(workflow, action), "http://localhost:8080/oozie"); assertNotNull(oozieClient); }
public void testActionCheck() throws Exception { JPAService jpaService = Services.get().get(JPAService.class); WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); WorkflowActionBean action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP); WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(action.getId()); new ActionStartXCommand(action.getId(), "map-reduce").call(); action = jpaService.execute(wfActionGetCmd); ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf())); String user = conf.get("user.name"); JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf); String launcherId = action.getExternalId(); final RunningJob launcherJob = jobClient.getJob(JobID.forName(launcherId)); waitFor( 120 * 1000, new Predicate() { public boolean evaluate() throws Exception { return launcherJob.isComplete(); } }); assertTrue(launcherJob.isSuccessful()); Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf); assertTrue(LauncherMapperHelper.hasIdSwap(actionData)); new ActionCheckXCommand(action.getId()).call(); action = jpaService.execute(wfActionGetCmd); String mapperId = action.getExternalId(); String childId = action.getExternalChildIDs(); assertTrue(launcherId.equals(mapperId)); final RunningJob mrJob = jobClient.getJob(JobID.forName(childId)); waitFor( 120 * 1000, new Predicate() { public boolean evaluate() throws Exception { return mrJob.isComplete(); } }); assertTrue(mrJob.isSuccessful()); new ActionCheckXCommand(action.getId()).call(); action = jpaService.execute(wfActionGetCmd); assertEquals("SUCCEEDED", action.getExternalStatus()); }
public void testSubworkflowLib() throws Exception { XConfiguration protoConf = getBaseProtoConf(); WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); FileSystem fs = getFileSystem(); Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar"); fs.create(parentLibJar); assertTrue(fs.exists(parentLibJar)); String defaultConf = workflow.getConf(); XConfiguration newConf = new XConfiguration(new StringReader(defaultConf)); newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString()); workflow.setConf(newConf.toXmlString()); Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf"); Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); writer.write(APP1); writer.close(); Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar"); fs.create(subwfLibJar); assertTrue(fs.exists(subwfLibJar)); final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>" + " <app-path>" + subWorkflowAppPath + File.separator + "workflow.xml" + "</app-path>" + "</sub-workflow>"); SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); subWorkflow.start(new Context(workflow, action), action); final OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); waitFor( JOB_TIMEOUT, new Predicate() { public boolean evaluate() throws Exception { return oozieClient.getJobInfo(action.getExternalId()).getStatus() == WorkflowJob.Status.SUCCEEDED; } }); assertEquals( WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus()); subWorkflow.check(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.DONE, action.getStatus()); subWorkflow.end(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.OK, action.getStatus()); WorkflowAppService wps = Services.get().get(WorkflowAppService.class); WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId()); Configuration childConf = new XConfiguration(new StringReader(wf.getConf())); childConf = wps.createProtoActionConf(childConf, "authToken", true); assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString()); }
@Override protected void verifyPrecondition() throws CommandException, PreconditionException { if ((wfAction == null) || (wfAction.isComplete() && wfAction.isPending())) { if (wfJob.getStatus() != WorkflowJob.Status.RUNNING && wfJob.getStatus() != WorkflowJob.Status.PREP) { throw new PreconditionException(ErrorCode.E0813, wfJob.getStatusStr()); } } else { throw new PreconditionException( ErrorCode.E0814, actionId, wfAction.getStatusStr(), wfAction.isPending()); } }
/** * Provides functionality to test kill node message * * @throws Exception */ public void testKillNodeErrorMessage() throws Exception { String workflowPath = getTestCaseFileUri("workflow.xml"); Reader reader = IOUtils.getResourceAsReader("wf-test-kill-node-message.xml", -1); Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml")); IOUtils.copyCharStream(reader, writer); final DagEngine engine = new DagEngine("u"); Configuration conf = new XConfiguration(); conf.set(OozieClient.APP_PATH, workflowPath); conf.set(OozieClient.USER_NAME, getTestUser()); conf.set(OozieClient.LOG_TOKEN, "t"); conf.set("error", "end.error"); conf.set("external-status", "FAILED/KILLED"); conf.set("signal-value", "fail"); final String jobId = engine.submitJob(conf, true); final JPAService jpaService = Services.get().get(JPAService.class); final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId); waitFor( 50000, new Predicate() { public boolean evaluate() throws Exception { WorkflowJobBean job = jpaService.execute(wfJobGetCmd); return (job.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED); } }); WorkflowJobBean job = jpaService.execute(wfJobGetCmd); assertEquals(WorkflowJob.Status.KILLED, job.getStatus()); WorkflowActionsGetForJobJPAExecutor wfActionsGetCmd = new WorkflowActionsGetForJobJPAExecutor(jobId); List<WorkflowActionBean> actions = jpaService.execute(wfActionsGetCmd); int n = actions.size(); WorkflowActionBean action = null; for (WorkflowActionBean bean : actions) { if (bean.getType().equals("test")) { action = bean; break; } } assertNotNull(action); assertEquals("TEST_ERROR", action.getErrorCode()); assertEquals("end", action.getErrorMessage()); assertEquals(WorkflowAction.Status.ERROR, action.getStatus()); }
@SuppressWarnings("unchecked") public void testSetupMethodsForQuery() throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); List<Class> classes = new ArrayList<Class>(); classes.add(Hive2Main.class); assertEquals(classes, ae.getLauncherClasses()); String sampleQuery = "SELECT count(*) from foobar"; Element actionXml = XmlUtils.parseXml( "<hive2 xmlns=\"uri:oozie:hive2-action:0.2\">" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<jdbc-url>jdbc:hive2://foo:1234/bar</jdbc-url>" + "<password>pass</password>" + "<query>" + sampleQuery + "</query>" + "<param>a=A</param>" + "<param>b=B</param>" + "<argument>-c</argument>" + "<argument>--dee</argument>" + "</hive2>"); XConfiguration protoConf = new XConfiguration(); protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action"); WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); action.setType(ae.getType()); Context context = new Context(wf, action); Configuration conf = ae.createBaseHadoopConf(context, actionXml); ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir()); assertEquals("jdbc:hive2://foo:1234/bar", conf.get("oozie.hive2.jdbc.url")); assertEquals("pass", conf.get("oozie.hive2.password")); assertEquals(sampleQuery, conf.get("oozie.hive2.query")); assertNull(conf.get("oozie.hive2.script")); assertEquals("2", conf.get("oozie.hive2.params.size")); assertEquals("a=A", conf.get("oozie.hive2.params.0")); assertEquals("b=B", conf.get("oozie.hive2.params.1")); assertEquals("2", conf.get("oozie.hive2.args.size")); assertEquals("-c", conf.get("oozie.hive2.args.0")); assertEquals("--dee", conf.get("oozie.hive2.args.1")); }
public void testExecuteUpdate() throws Exception { WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); WorkflowActionBean bean = addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP); bean.setStatus(WorkflowAction.Status.RUNNING); bean.setName("test-name"); WorkflowActionQueryExecutor.getInstance() .executeUpdate(WorkflowActionQuery.UPDATE_ACTION, bean); WorkflowActionBean retBean = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION, bean.getId()); assertEquals("test-name", retBean.getName()); assertEquals(retBean.getStatus(), WorkflowAction.Status.RUNNING); }
private Context createContext(String actionXml) throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); XConfiguration protoConf = new XConfiguration(); protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); SharelibUtils.addToDistributedCache("hive2", getFileSystem(), getFsTestCaseDir(), protoConf); WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action"); WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); action.setType(ae.getType()); action.setConf(actionXml); return new Context(wf, action); }
/** * v1 service implementation to get a list of workflows, with filtering or interested windows * embedded in the request object */ private JSONObject getWorkflowJobs(HttpServletRequest request) throws XServletException { JSONObject json = new JSONObject(); try { String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM); String startStr = request.getParameter(RestConstants.OFFSET_PARAM); String lenStr = request.getParameter(RestConstants.LEN_PARAM); int start = (startStr != null) ? Integer.parseInt(startStr) : 1; start = (start < 1) ? 1 : start; int len = (lenStr != null) ? Integer.parseInt(lenStr) : 50; len = (len < 1) ? 50 : len; DagEngine dagEngine = Services.get() .get(DagEngineService.class) .getDagEngine(getUser(request), getAuthToken(request)); WorkflowsInfo jobs = dagEngine.getJobs(filter, start, len); List<WorkflowJobBean> jsonWorkflows = jobs.getWorkflows(); json.put(JsonTags.WORKFLOWS_JOBS, WorkflowJobBean.toJSONArray(jsonWorkflows)); json.put(JsonTags.WORKFLOWS_TOTAL, jobs.getTotal()); json.put(JsonTags.WORKFLOWS_OFFSET, jobs.getStart()); json.put(JsonTags.WORKFLOWS_LEN, jobs.getLen()); } catch (DagEngineException ex) { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex); } return json; }
/** * This test case verifies if getRetryInterval picks up the overridden value. * * @throws Exception */ public void testCheckInterval() throws Exception { long testedValue = 10; Services.get().getConf().setLong(ActionCheckerService.CONF_ACTION_CHECK_INTERVAL, testedValue); WorkflowJobBean job0 = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); final String jobId = job0.getId(); WorkflowActionBean action0 = this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.RUNNING); final String actionId = action0.getId(); ActionCheckXCommand checkCommand = new ActionCheckXCommand(actionId); checkCommand.call(); long effectiveValue = checkCommand.getRetryInterval(); assertEquals(testedValue, effectiveValue); }
protected void updateSLASummary() throws XException { updateSLASummary(wfJob.getId(), wfJob.getStartTime(), wfJob.getEndTime(), wfJob.getStatusStr()); }
/** * Check if the user+group is authorized to operate on the specified job. * * <p>Checks if the user is a super-user or the one who started the job. * * <p>Read operations are allowed to all users. * * @param user user name. * @param jobId job id. * @param write indicates if the check is for read or write job tasks. * @throws AuthorizationException thrown if the user is not authorized for the job. */ public void authorizeForJob(String user, String jobId, boolean write) throws AuthorizationException { if (authorizationEnabled && write && !isAdmin(user)) { try { // handle workflow jobs if (jobId.endsWith("-W")) { WorkflowJobBean jobBean = null; JPAService jpaService = Services.get().get(JPAService.class); if (jpaService != null) { try { jobBean = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId)); } catch (JPAExecutorException je) { throw new AuthorizationException(je); } } else { throw new AuthorizationException(ErrorCode.E0610); } if (jobBean != null && !jobBean.getUser().equals(user)) { if (!isUserInAcl(user, jobBean.getGroup())) { incrCounter(INSTR_FAILED_AUTH_COUNTER, 1); throw new AuthorizationException(ErrorCode.E0508, user, jobId); } } } // handle bundle jobs else if (jobId.endsWith("-B")) { BundleJobBean jobBean = null; JPAService jpaService = Services.get().get(JPAService.class); if (jpaService != null) { try { jobBean = jpaService.execute(new BundleJobGetJPAExecutor(jobId)); } catch (JPAExecutorException je) { throw new AuthorizationException(je); } } else { throw new AuthorizationException(ErrorCode.E0610); } if (jobBean != null && !jobBean.getUser().equals(user)) { if (!isUserInAcl(user, jobBean.getGroup())) { incrCounter(INSTR_FAILED_AUTH_COUNTER, 1); throw new AuthorizationException(ErrorCode.E0509, user, jobId); } } } // handle coordinator jobs else { CoordinatorJobBean jobBean = null; JPAService jpaService = Services.get().get(JPAService.class); if (jpaService != null) { try { jobBean = jpaService.execute(new CoordJobGetJPAExecutor(jobId)); } catch (JPAExecutorException je) { throw new AuthorizationException(je); } } else { throw new AuthorizationException(ErrorCode.E0610); } if (jobBean != null && !jobBean.getUser().equals(user)) { if (!isUserInAcl(user, jobBean.getGroup())) { incrCounter(INSTR_FAILED_AUTH_COUNTER, 1); throw new AuthorizationException(ErrorCode.E0509, user, jobId); } } } } catch (IOException ex) { throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex); } } }
/** * Provides functionality to test non transient failures and coordinator action update * * @param errorType the error type. (start.non-transient, end.non-transient) * @param expStatus1 expected status. (START_MANUAL, END_MANUAL) * @param expErrorMsg expected error message. * @throws Exception */ private void _testNonTransientWithCoordActionUpdate( String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception { String workflowPath = getTestCaseFileUri("workflow.xml"); Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml")); IOUtils.copyCharStream(reader, writer); final DagEngine engine = new DagEngine("u"); Configuration conf = new XConfiguration(); conf.set(OozieClient.APP_PATH, workflowPath); conf.set(OozieClient.USER_NAME, getTestUser()); conf.set(OozieClient.LOG_TOKEN, "t"); conf.set("signal-value", "OK"); conf.set("external-status", "ok"); conf.set("error", errorType); final String jobId = engine.submitJob(conf, false); final JPAService jpaService = Services.get().get(JPAService.class); final CoordinatorJobBean coordJob = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, false, false); CoordinatorActionBean coordAction = addRecordToCoordActionTable( coordJob.getId(), 1, CoordinatorAction.Status.RUNNING, "coord-action-get.xml", jobId, "RUNNING", 0); engine.start(jobId); waitFor( 5000, new Predicate() { public boolean evaluate() throws Exception { return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED); } }); assertNotNull(jpaService); WorkflowJobGetJPAExecutor wfGetCmd = new WorkflowJobGetJPAExecutor(jobId); WorkflowJobBean job = jpaService.execute(wfGetCmd); WorkflowActionsGetForJobJPAExecutor actionsGetExe = new WorkflowActionsGetForJobJPAExecutor(jobId); List<WorkflowActionBean> actionsList = jpaService.execute(actionsGetExe); int n = actionsList.size(); WorkflowActionBean action = actionsList.get(n - 1); assertEquals("TEST_ERROR", action.getErrorCode()); assertEquals(expErrorMsg, action.getErrorMessage()); assertEquals(expStatus1, action.getStatus()); assertFalse(action.isPending()); assertEquals(WorkflowJob.Status.SUSPENDED, job.getStatus()); waitFor( 5000, new Predicate() { public boolean evaluate() throws Exception { CoordinatorActionBean coordAction2 = jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId)); return coordAction2.getStatus().equals(CoordinatorAction.Status.SUSPENDED); } }); coordAction = jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId)); assertEquals(CoordinatorAction.Status.SUSPENDED, coordAction.getStatus()); }
public void testActionCheckTransientDuringMRAction() throws Exception { // When using YARN, skip this test because it relies on shutting down the job tracker, which // isn't used in YARN if (createJobConf().get("yarn.resourcemanager.address") != null) { return; } services.destroy(); // Make the max number of retries lower so the test won't take as long final int maxRetries = 2; setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries)); services = new Services(); // Disable ActionCheckerService so it doesn't interfere by triggering any extra // ActionCheckXCommands setClassesToBeExcluded( services.getConf(), new String[] {"org.apache.oozie.service.ActionCheckerService"}); services.init(); final JPAService jpaService = Services.get().get(JPAService.class); WorkflowJobBean job0 = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); final String jobId = job0.getId(); WorkflowActionBean action0 = this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP); final String actionId = action0.getId(); final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId); new ActionStartXCommand(actionId, "map-reduce").call(); final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd); String originalLauncherId = action1.getExternalId(); ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job0, action1, false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf())); String user = conf.get("user.name"); JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf); final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId)); waitFor( 120 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { return launcherJob.isComplete(); } }); assertTrue(launcherJob.isSuccessful()); Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf); assertTrue(LauncherMapperHelper.hasIdSwap(actionData)); new ActionCheckXCommand(action1.getId()).call(); WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd); String originalMapperId = action2.getExternalChildIDs(); assertFalse(originalLauncherId.equals(originalMapperId)); // At this point, the launcher job has finished and the map-reduce action has started (but not // finished) // Now, shutdown the job tracker to pretend it has gone down during the map-reduce job executeWhileJobTrackerIsShutdown( new ShutdownJobTrackerExecutable() { @Override public void execute() throws Exception { assertEquals(0, action1.getRetries()); new ActionCheckXCommand(actionId).call(); waitFor( 30 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd); return (action1a.getRetries() > 0); } }); waitFor( 180 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd); return (action1a.getRetries() == 0); } }); WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd); assertEquals(0, action1b.getRetries()); assertEquals("START_MANUAL", action1b.getStatusStr()); WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId)); assertEquals("SUSPENDED", job1.getStatusStr()); // At this point, the action has gotten a transient error, even after maxRetries tries // so the workflow has been // SUSPENDED } }); // Now, lets bring the job tracker back up and resume the workflow (which will restart the // current action) // It should now continue and finish with SUCCEEDED new ResumeXCommand(jobId).call(); WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId)); assertEquals("RUNNING", job2.getStatusStr()); sleep(500); new ActionCheckXCommand(actionId).call(); WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd); String launcherId = action3.getExternalId(); assertFalse(originalLauncherId.equals(launcherId)); final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId)); waitFor( 120 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { return launcherJob2.isComplete(); } }); assertTrue(launcherJob2.isSuccessful()); actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf); assertTrue(LauncherMapperHelper.hasIdSwap(actionData)); new ActionCheckXCommand(actionId).call(); WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd); String mapperId = action4.getExternalChildIDs(); assertFalse(originalMapperId.equals(mapperId)); final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId)); waitFor( 120 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { return mrJob.isComplete(); } }); assertTrue(mrJob.isSuccessful()); new ActionCheckXCommand(actionId).call(); WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd); assertEquals("SUCCEEDED", action5.getExternalStatus()); }
protected Void call(WorkflowStore store) throws StoreException, CommandException { WorkflowJobBean workflow = store.getWorkflow(jobId, false); setLogInfo(workflow); WorkflowActionBean action = store.getAction(id, false); setLogInfo(action); if (action.isPending() && (action.getStatus() == WorkflowActionBean.Status.DONE || action.getStatus() == WorkflowActionBean.Status.END_RETRY || action.getStatus() == WorkflowActionBean.Status.END_MANUAL)) { if (workflow.getStatus() == WorkflowJob.Status.RUNNING) { ActionExecutor executor = Services.get().get(ActionService.class).getExecutor(action.getType()); Configuration conf = workflow.getWorkflowInstance().getConf(); int maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries()); long retryInterval = conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval()); executor.setMaxRetries(maxRetries); executor.setRetryInterval(retryInterval); if (executor != null) { boolean isRetry = false; if (action.getStatus() == WorkflowActionBean.Status.END_RETRY || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) { isRetry = true; } ActionExecutorContext context = new ActionCommand.ActionExecutorContext(workflow, action, isRetry); try { XLog.getLog(getClass()) .debug( "End, name [{0}] type [{1}] status[{2}] external status [{3}] signal value [{4}]", action.getName(), action.getType(), action.getStatus(), action.getExternalStatus(), action.getSignalValue()); WorkflowInstance wfInstance = workflow.getWorkflowInstance(); DagELFunctions.setActionInfo(wfInstance, action); workflow.setWorkflowInstance(wfInstance); incrActionCounter(action.getType(), 1); Instrumentation.Cron cron = new Instrumentation.Cron(); cron.start(); executor.end(context, action); cron.stop(); addActionCron(action.getType(), cron); if (!context.isEnded()) { XLog.getLog(getClass()) .warn( XLog.OPS, "Action Ended, ActionExecutor [{0}] must call setEndData()", executor.getType()); action.setErrorInfo( END_DATA_MISSING, "Execution Ended, but End Data Missing from Action"); failJob(context); store.updateAction(action); store.updateWorkflow(workflow); return null; } action.setRetries(0); action.setEndTime(new Date()); store.updateAction(action); store.updateWorkflow(workflow); Status slaStatus = null; switch (action.getStatus()) { case OK: slaStatus = Status.SUCCEEDED; break; case KILLED: slaStatus = Status.KILLED; break; case FAILED: slaStatus = Status.FAILED; break; case ERROR: XLog.getLog(getClass()).info("ERROR is considered as FAILED for SLA"); slaStatus = Status.KILLED; break; default: // TODO: What will happen for other Action // status slaStatus = Status.FAILED; break; } SLADbOperations.writeStausEvent( action.getSlaXml(), action.getId(), store, slaStatus, SlaAppType.WORKFLOW_ACTION); queueCallable(new NotificationCommand(workflow, action)); XLog.getLog(getClass()) .debug( "Queuing commands for action " + id + " status " + action.getStatus() + ", Set pending=" + action.getPending()); queueCallable(new SignalCommand(workflow.getId(), id)); } catch (ActionExecutorException ex) { XLog.getLog(getClass()) .warn( "Error ending action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]", action.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage()); action.setErrorInfo(ex.getErrorCode(), ex.getMessage()); action.setEndTime(null); switch (ex.getErrorType()) { case TRANSIENT: if (!handleTransient(context, executor, WorkflowAction.Status.END_RETRY)) { handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL); action.setPendingAge(new Date()); action.setRetries(0); } action.setEndTime(null); break; case NON_TRANSIENT: handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL); action.setEndTime(null); break; case ERROR: handleError(context, executor, COULD_NOT_END, false, WorkflowAction.Status.ERROR); queueCallable(new SignalCommand(workflow.getId(), id)); break; case FAILED: failJob(context); break; } store.updateAction(action); store.updateWorkflow(workflow); } } else { throw new CommandException(ErrorCode.E0802, action.getType()); } } else { XLog.getLog(getClass()) .warn( "Job state is not {0}. Skipping ActionEnd Execution", WorkflowJob.Status.RUNNING.toString()); } } else { XLog.getLog(getClass()) .debug( "Action pending={0}, status={1}. Skipping ActionEnd Execution", action.getPending(), action.getStatusStr()); } return null; }
/** * Test : verify the PreconditionException is thrown when job != RUNNING && job != SUSPENDED * * @throws Exception */ public void testActionCheckPreCondition4() throws Exception { Instrumentation inst = Services.get().get(InstrumentationService.class).get(); WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.FAILED, WorkflowInstance.Status.FAILED); WorkflowActionBean action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING); ActionCheckXCommand checkCmd = new ActionCheckXCommand(action.getId()); long counterVal; try { counterVal = inst.getCounters() .get(XCommand.INSTRUMENTATION_GROUP) .get(checkCmd.getName() + ".preconditionfailed") .getValue(); } catch (NullPointerException e) { // counter might be null counterVal = 0L; } assertEquals(0L, counterVal); checkCmd.call(); // precondition failed because of job != RUNNING && job != SUSPENDED counterVal = inst.getCounters() .get(XCommand.INSTRUMENTATION_GROUP) .get(checkCmd.getName() + ".preconditionfailed") .getValue(); assertEquals(1L, counterVal); job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING); checkCmd = new ActionCheckXCommand(action.getId()); checkCmd.call(); // precondition passed because job == RUNNING so counter shouldn't have incremented counterVal = inst.getCounters() .get(XCommand.INSTRUMENTATION_GROUP) .get(checkCmd.getName() + ".preconditionfailed") .getValue(); assertEquals(1L, counterVal); job = this.addRecordToWfJobTable(WorkflowJob.Status.SUSPENDED, WorkflowInstance.Status.SUSPENDED); action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING); checkCmd = new ActionCheckXCommand(action.getId()); checkCmd.call(); // precondition passed because job == SUSPENDED so counter shouldn't have incremented counterVal = inst.getCounters() .get(XCommand.INSTRUMENTATION_GROUP) .get(checkCmd.getName() + ".preconditionfailed") .getValue(); assertEquals(1L, counterVal); job = this.addRecordToWfJobTable(WorkflowJob.Status.SUCCEEDED, WorkflowInstance.Status.SUCCEEDED); action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING); checkCmd = new ActionCheckXCommand(action.getId()); checkCmd.call(); // precondition failed because of job != RUNNING && job != SUSPENDED counterVal = inst.getCounters() .get(XCommand.INSTRUMENTATION_GROUP) .get(checkCmd.getName() + ".preconditionfailed") .getValue(); assertEquals(2L, counterVal); job = this.addRecordToWfJobTable(WorkflowJob.Status.KILLED, WorkflowInstance.Status.KILLED); action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING); checkCmd = new ActionCheckXCommand(action.getId()); checkCmd.call(); // precondition failed because of job != RUNNING && job != SUSPENDED counterVal = inst.getCounters() .get(XCommand.INSTRUMENTATION_GROUP) .get(checkCmd.getName() + ".preconditionfailed") .getValue(); assertEquals(3L, counterVal); }
public void testGetUpdateQuery() throws Exception { EntityManager em = jpaService.getEntityManager(); WorkflowJobBean job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING); WorkflowActionBean bean = addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP); // UPDATE_ACTION Query query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em); assertEquals(query.getParameterValue("conf"), bean.getConfBlob()); assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl()); assertEquals(query.getParameterValue("data"), bean.getDataBlob()); assertEquals(query.getParameterValue("stats"), bean.getStatsBlob()); assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob()); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("externalId"), bean.getExternalId()); assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus()); assertEquals(query.getParameterValue("name"), bean.getName()); assertEquals(query.getParameterValue("cred"), bean.getCred()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri()); assertEquals(query.getParameterValue("transition"), bean.getTransition()); assertEquals(query.getParameterValue("type"), bean.getType()); assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp()); assertEquals(query.getParameterValue("executionPath"), bean.getExecutionPath()); assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp()); assertEquals(query.getParameterValue("logToken"), bean.getLogToken()); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAge()); assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue()); assertEquals(query.getParameterValue("slaXml"), bean.getSlaXmlBlob()); assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("wfId"), bean.getWfId()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_FOR_LAST_CHECKED_TIME query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em); assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_PENDING query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING, bean, em); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_STATUS_PENDING query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, bean, em); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_PENDING_TRANS query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS, bean, em); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("transition"), bean.getTransition()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_PENDING_TRANS_ERROR query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS_ERROR, bean, em); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("transition"), bean.getTransition()); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_START query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_START, bean, em); assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp()); assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob()); assertEquals(query.getParameterValue("conf"), bean.getConfBlob()); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("externalId"), bean.getExternalId()); assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri()); assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl()); assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus()); assertEquals(query.getParameterValue("data"), bean.getDataBlob()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_CHECK query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_CHECK, bean, em); assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob()); assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("data"), bean.getDataBlob()); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp()); assertEquals(query.getParameterValue("stats"), bean.getStatsBlob()); assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount()); assertEquals(query.getParameterValue("id"), bean.getId()); // UPDATE_ACTION_END query = WorkflowActionQueryExecutor.getInstance() .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_END, bean, em); assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode()); assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp()); assertEquals(query.getParameterValue("status"), bean.getStatus().toString()); assertEquals(query.getParameterValue("retries"), bean.getRetries()); assertEquals(query.getParameterValue("pending"), bean.getPending()); assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp()); assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue()); assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount()); assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus()); assertEquals(query.getParameterValue("stats"), bean.getStatsBlob()); assertEquals(query.getParameterValue("id"), bean.getId()); em.close(); }
public void testSubWorkflowRecovery() throws Exception { Path subWorkflowAppPath = getFsTestCaseDir(); FileSystem fs = getFileSystem(); Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); writer.write(APP1); writer.close(); XConfiguration protoConf = getBaseProtoConf(); WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1'>" + " <app-path>" + subWorkflowAppPath + File.separator + "workflow.xml" + "</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"); SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); subWorkflow.start(new Context(workflow, action), action); final OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); waitFor( JOB_TIMEOUT, new Predicate() { public boolean evaluate() throws Exception { return oozieClient.getJobInfo(action.getExternalId()).getStatus() == WorkflowJob.Status.SUCCEEDED; } }); String extId = action.getExternalId(); assertEquals(WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(extId).getStatus()); WorkflowActionBean action1 = new WorkflowActionBean(); action1.setId(action.getId()); action1.setName(action.getName()); action1.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1'>" + " <app-path>wrongAppPath</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"); subWorkflow.start(new Context(workflow, action1), action1); assertEquals(extId, action1.getExternalId()); subWorkflow.check(new Context(workflow, action1), action1); assertEquals(WorkflowAction.Status.DONE, action1.getStatus()); subWorkflow.end(new Context(workflow, action1), action1); assertEquals(WorkflowAction.Status.OK, action1.getStatus()); }
@Override protected Void execute() throws CommandException { LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId); WorkflowInstance workflowInstance = wfJob.getWorkflowInstance(); workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob); boolean completed = false; boolean skipAction = false; if (wfAction == null) { if (wfJob.getStatus() == WorkflowJob.Status.PREP) { try { completed = workflowInstance.start(); } catch (WorkflowException e) { throw new CommandException(e); } wfJob.setStatus(WorkflowJob.Status.RUNNING); wfJob.setStartTime(new Date()); wfJob.setWorkflowInstance(workflowInstance); // 1. Add SLA status event for WF-JOB with status STARTED // 2. Add SLA registration events for all WF_ACTIONS SLADbXOperations.writeStausEvent( wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB); writeSLARegistrationForAllActions( workflowInstance.getApp().getDefinition(), wfJob.getUser(), wfJob.getGroup(), wfJob.getConf()); queue(new NotificationXCommand(wfJob)); } else { throw new CommandException(ErrorCode.E0801, wfJob.getId()); } } else { String skipVar = workflowInstance.getVar( wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP); if (skipVar != null) { skipAction = skipVar.equals("true"); } try { completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue()); } catch (WorkflowException e) { throw new CommandException(e); } wfJob.setWorkflowInstance(workflowInstance); wfAction.resetPending(); if (!skipAction) { wfAction.setTransition(workflowInstance.getTransition(wfAction.getName())); } try { jpaService.execute(new WorkflowActionUpdateJPAExecutor(wfAction)); } catch (JPAExecutorException je) { throw new CommandException(je); } } if (completed) { try { for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) { WorkflowActionBean actionToKill; actionToKill = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToKillId)); actionToKill.setPending(); actionToKill.setStatus(WorkflowActionBean.Status.KILLED); jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToKill)); queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType())); } for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) { WorkflowActionBean actionToFail = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToFailId)); actionToFail.resetPending(); actionToFail.setStatus(WorkflowActionBean.Status.FAILED); SLADbXOperations.writeStausEvent( wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION); jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToFail)); } } catch (JPAExecutorException je) { throw new CommandException(je); } wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString())); wfJob.setEndTime(new Date()); wfJob.setWorkflowInstance(workflowInstance); Status slaStatus = Status.SUCCEEDED; switch (wfJob.getStatus()) { case SUCCEEDED: slaStatus = Status.SUCCEEDED; break; case KILLED: slaStatus = Status.KILLED; break; case FAILED: slaStatus = Status.FAILED; break; default: // TODO SUSPENDED break; } SLADbXOperations.writeStausEvent( wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB); queue(new NotificationXCommand(wfJob)); if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) { InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation()); } } else { for (WorkflowActionBean newAction : WorkflowStoreService.getStartedActions(workflowInstance)) { String skipVar = workflowInstance.getVar( newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP); boolean skipNewAction = false; if (skipVar != null) { skipNewAction = skipVar.equals("true"); } try { if (skipNewAction) { WorkflowActionBean oldAction; oldAction = jpaService.execute(new WorkflowActionGetJPAExecutor(newAction.getId())); oldAction.setPending(); jpaService.execute(new WorkflowActionUpdateJPAExecutor(oldAction)); queue(new SignalXCommand(jobId, oldAction.getId())); } else { newAction.setPending(); String actionSlaXml = getActionSLAXml( newAction.getName(), workflowInstance.getApp().getDefinition(), wfJob.getConf()); newAction.setSlaXml(actionSlaXml); jpaService.execute(new WorkflowActionInsertJPAExecutor(newAction)); LOG.debug( "SignalXCommand: Name: " + newAction.getName() + ", Id: " + newAction.getId() + ", Authcode:" + newAction.getCred()); queue(new ActionStartXCommand(newAction.getId(), newAction.getType())); } } catch (JPAExecutorException je) { throw new CommandException(je); } } } try { jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob)); } catch (JPAExecutorException je) { throw new CommandException(je); } XLog.getLog(getClass()) .debug( "Updated the workflow status to " + wfJob.getId() + " status =" + wfJob.getStatusStr()); if (wfJob.getStatus() != WorkflowJob.Status.RUNNING && wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) { // update coordinator action new CoordActionUpdateXCommand(wfJob).call(); new WfEndXCommand(wfJob).call(); // To delete the WF temp dir } LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId); return null; }
public void testGetGroupFromParent() throws Exception { Path subWorkflowAppPath = getFsTestCaseDir(); FileSystem fs = getFileSystem(); Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); writer.write(APP1); writer.close(); XConfiguration protoConf = getBaseProtoConf(); final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); String defaultConf = workflow.getConf(); XConfiguration newConf = new XConfiguration(new StringReader(defaultConf)); String actionConf = "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>" + " <app-path>" + subWorkflowAppPath + File.separator + "workflow.xml" + "</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"; final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf(actionConf); // negative test final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); workflow.setConf(newConf.toXmlString()); subWorkflow.start(new Context(workflow, action), action); OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); waitFor( 5000, new Predicate() { @Override public boolean evaluate() throws Exception { subWorkflow.check(new Context(workflow, action), action); return action.getStatus() == WorkflowActionBean.Status.DONE; } }); subWorkflow.check(new Context(workflow, action), action); subWorkflow.end(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.OK, action.getStatus()); WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId()); Configuration childConf = new XConfiguration(new StringReader(wf.getConf())); assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME)); // positive test newConf.set(OozieClient.GROUP_NAME, getTestGroup()); workflow.setConf(newConf.toXmlString()); final WorkflowActionBean action1 = new WorkflowActionBean(); action1.setConf(actionConf); action1.setId("W1"); subWorkflow.start(new Context(workflow, action1), action1); oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL); waitFor( 5000, new Predicate() { @Override public boolean evaluate() throws Exception { subWorkflow.check(new Context(workflow, action1), action1); return action1.getStatus() == WorkflowActionBean.Status.DONE; } }); subWorkflow.check(new Context(workflow, action1), action1); subWorkflow.end(new Context(workflow, action1), action1); wf = oozieClient.getJobInfo(action1.getExternalId()); childConf = new XConfiguration(new StringReader(wf.getConf())); assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME)); }
@Override protected boolean isJobEnded() { return wfJob.inTerminalState(); }