public void testConfigNotPropagation() throws Exception { Path subWorkflowAppPath = getFsTestCaseDir(); FileSystem fs = getFileSystem(); Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); writer.write(APP1); writer.close(); XConfiguration protoConf = getBaseProtoConf(); WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); String defaultConf = workflow.getConf(); XConfiguration newConf = new XConfiguration(new StringReader(defaultConf)); newConf.set("abc", "xyz"); workflow.setConf(newConf.toXmlString()); final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>" + " <app-path>" + subWorkflowAppPath + File.separator + "workflow.xml" + "</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"); SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); subWorkflow.start(new Context(workflow, action), action); final OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); waitFor( JOB_TIMEOUT, new Predicate() { public boolean evaluate() throws Exception { return oozieClient.getJobInfo(action.getExternalId()).getStatus() == WorkflowJob.Status.SUCCEEDED; } }); assertEquals( WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus()); subWorkflow.check(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.DONE, action.getStatus()); subWorkflow.end(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.OK, action.getStatus()); WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId()); Configuration childConf = new XConfiguration(new StringReader(wf.getConf())); assertNull(childConf.get("abc")); }
public void testSubWorkflowConfCreation() throws Exception { SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); XConfiguration protoConf = getBaseProtoConf(); WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1'>" + " <app-path>hdfs://foo:9000/user/bar/workflow.xml</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"); OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); assertNotNull(oozieClient); oozieClient = subWorkflow.getWorkflowClient(new Context(workflow, action), "http://localhost:8080/oozie"); assertNotNull(oozieClient); }
public void testSubworkflowLib() throws Exception { XConfiguration protoConf = getBaseProtoConf(); WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); FileSystem fs = getFileSystem(); Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar"); fs.create(parentLibJar); assertTrue(fs.exists(parentLibJar)); String defaultConf = workflow.getConf(); XConfiguration newConf = new XConfiguration(new StringReader(defaultConf)); newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString()); workflow.setConf(newConf.toXmlString()); Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf"); Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); writer.write(APP1); writer.close(); Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar"); fs.create(subwfLibJar); assertTrue(fs.exists(subwfLibJar)); final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>" + " <app-path>" + subWorkflowAppPath + File.separator + "workflow.xml" + "</app-path>" + "</sub-workflow>"); SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); subWorkflow.start(new Context(workflow, action), action); final OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); waitFor( JOB_TIMEOUT, new Predicate() { public boolean evaluate() throws Exception { return oozieClient.getJobInfo(action.getExternalId()).getStatus() == WorkflowJob.Status.SUCCEEDED; } }); assertEquals( WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus()); subWorkflow.check(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.DONE, action.getStatus()); subWorkflow.end(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.OK, action.getStatus()); WorkflowAppService wps = Services.get().get(WorkflowAppService.class); WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId()); Configuration childConf = new XConfiguration(new StringReader(wf.getConf())); childConf = wps.createProtoActionConf(childConf, "authToken", true); assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString()); }
protected WorkflowActionBean createWorkflowActionSetPending( String wfId, WorkflowAction.Status status) throws Exception { WorkflowActionBean action = new WorkflowActionBean(); String actionname = "testAction"; action.setName(actionname); action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname)); action.setJobId(wfId); action.setType("map-reduce"); action.setTransition("transition"); action.setStatus(status); action.setStartTime(new Date()); action.setEndTime(new Date()); action.setLastCheckTime(new Date()); action.setPending(); action.setExecutionPath("/"); Path inputDir = new Path(getFsTestCaseDir(), "input"); Path outputDir = new Path(getFsTestCaseDir(), "output"); FileSystem fs = getFileSystem(); Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt"))); w.write("dummy\n"); w.write("dummy\n"); w.close(); String actionXml = "<map-reduce>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<prepare><delete path=\"" + outputDir.toString() + "\"/></prepare>" + "<configuration>" + "<property><name>mapred.mapper.class</name><value>" + MapperReducerForTest.class.getName() + "</value></property>" + "<property><name>mapred.reducer.class</name><value>" + MapperReducerForTest.class.getName() + "</value></property>" + "<property><name>mapred.input.dir</name><value>" + inputDir.toString() + "</value></property>" + "<property><name>mapred.output.dir</name><value>" + outputDir.toString() + "</value></property>" + "</configuration>" + "</map-reduce>"; action.setConf(actionXml); return action; }
private Context createContext(String actionXml) throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); XConfiguration protoConf = new XConfiguration(); protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); SharelibUtils.addToDistributedCache("hive2", getFileSystem(), getFsTestCaseDir(), protoConf); WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action"); WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); action.setType(ae.getType()); action.setConf(actionXml); return new Context(wf, action); }
public void testGetGroupFromParent() throws Exception { Path subWorkflowAppPath = getFsTestCaseDir(); FileSystem fs = getFileSystem(); Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); writer.write(APP1); writer.close(); XConfiguration protoConf = getBaseProtoConf(); final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); String defaultConf = workflow.getConf(); XConfiguration newConf = new XConfiguration(new StringReader(defaultConf)); String actionConf = "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>" + " <app-path>" + subWorkflowAppPath + File.separator + "workflow.xml" + "</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"; final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf(actionConf); // negative test final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); workflow.setConf(newConf.toXmlString()); subWorkflow.start(new Context(workflow, action), action); OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); waitFor( 5000, new Predicate() { @Override public boolean evaluate() throws Exception { subWorkflow.check(new Context(workflow, action), action); return action.getStatus() == WorkflowActionBean.Status.DONE; } }); subWorkflow.check(new Context(workflow, action), action); subWorkflow.end(new Context(workflow, action), action); assertEquals(WorkflowAction.Status.OK, action.getStatus()); WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId()); Configuration childConf = new XConfiguration(new StringReader(wf.getConf())); assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME)); // positive test newConf.set(OozieClient.GROUP_NAME, getTestGroup()); workflow.setConf(newConf.toXmlString()); final WorkflowActionBean action1 = new WorkflowActionBean(); action1.setConf(actionConf); action1.setId("W1"); subWorkflow.start(new Context(workflow, action1), action1); oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL); waitFor( 5000, new Predicate() { @Override public boolean evaluate() throws Exception { subWorkflow.check(new Context(workflow, action1), action1); return action1.getStatus() == WorkflowActionBean.Status.DONE; } }); subWorkflow.check(new Context(workflow, action1), action1); subWorkflow.end(new Context(workflow, action1), action1); wf = oozieClient.getJobInfo(action1.getExternalId()); childConf = new XConfiguration(new StringReader(wf.getConf())); assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME)); }
public void testSubWorkflowRecovery() throws Exception { Path subWorkflowAppPath = getFsTestCaseDir(); FileSystem fs = getFileSystem(); Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); writer.write(APP1); writer.close(); XConfiguration protoConf = getBaseProtoConf(); WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W"); final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0); action.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1'>" + " <app-path>" + subWorkflowAppPath + File.separator + "workflow.xml" + "</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"); SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); subWorkflow.start(new Context(workflow, action), action); final OozieClient oozieClient = subWorkflow.getWorkflowClient( new Context(workflow, action), SubWorkflowActionExecutor.LOCAL); waitFor( JOB_TIMEOUT, new Predicate() { public boolean evaluate() throws Exception { return oozieClient.getJobInfo(action.getExternalId()).getStatus() == WorkflowJob.Status.SUCCEEDED; } }); String extId = action.getExternalId(); assertEquals(WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(extId).getStatus()); WorkflowActionBean action1 = new WorkflowActionBean(); action1.setId(action.getId()); action1.setName(action.getName()); action1.setConf( "<sub-workflow xmlns='uri:oozie:workflow:0.1'>" + " <app-path>wrongAppPath</app-path>" + " <configuration>" + " <property>" + " <name>a</name>" + " <value>A</value>" + " </property>" + " </configuration>" + "</sub-workflow>"); subWorkflow.start(new Context(workflow, action1), action1); assertEquals(extId, action1.getExternalId()); subWorkflow.check(new Context(workflow, action1), action1); assertEquals(WorkflowAction.Status.DONE, action1.getStatus()); subWorkflow.end(new Context(workflow, action1), action1); assertEquals(WorkflowAction.Status.OK, action1.getStatus()); }
/** * Provides functionality to test non transient failures. * * @param errorType the error type. (start.non-transient, end.non-transient) * @param expStatus1 expected status. (START_MANUAL, END_MANUAL) * @param expErrorMsg expected error message. * @throws Exception */ private void _testNonTransient( String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception { String workflowPath = getTestCaseFileUri("workflow.xml"); Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml")); IOUtils.copyCharStream(reader, writer); final DagEngine engine = new DagEngine("u"); Configuration conf = new XConfiguration(); conf.set(OozieClient.APP_PATH, workflowPath); conf.set(OozieClient.USER_NAME, getTestUser()); conf.set(OozieClient.LOG_TOKEN, "t"); conf.set("signal-value", "OK"); conf.set("external-status", "ok"); conf.set("error", errorType); final String jobId = engine.submitJob(conf, true); waitFor( 5000, new Predicate() { public boolean evaluate() throws Exception { return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED); } }); final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); store.beginTrx(); List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, true); int n = actions.size(); WorkflowActionBean action = actions.get(n - 1); assertEquals("TEST_ERROR", action.getErrorCode()); assertEquals(expErrorMsg, action.getErrorMessage()); assertEquals(expStatus1, action.getStatus()); assertTrue(action.isPending() == false); assertTrue(engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED); String actionConf = action.getConf(); String fixedActionConf = actionConf.replaceAll(errorType, "none"); action.setConf(fixedActionConf); store.updateAction(action); store.commitTrx(); store.closeTrx(); engine.resume(jobId); waitFor( 5000, new Predicate() { public boolean evaluate() throws Exception { return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED); } }); assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus()); final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create(); store2.beginTrx(); actions = store2.getActionsForWorkflow(jobId, false); action = actions.get(0); assertEquals(null, action.getErrorCode()); assertEquals(null, action.getErrorMessage()); assertEquals(WorkflowActionBean.Status.OK, action.getStatus()); store2.commitTrx(); store2.closeTrx(); }