public void testConfigNotPropagation() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set("abc", "xyz");
workflow.setConf(newConf.toXmlString());
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
assertNull(childConf.get("abc"));
}
public void testSubWorkflowConfCreation() throws Exception {
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
+ " <app-path>hdfs://foo:9000/user/bar/workflow.xml</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
assertNotNull(oozieClient);
oozieClient =
subWorkflow.getWorkflowClient(new Context(workflow, action), "http://localhost:8080/oozie");
assertNotNull(oozieClient);
}
public void testSubworkflowLib() throws Exception {
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
FileSystem fs = getFileSystem();
Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
fs.create(parentLibJar);
assertTrue(fs.exists(parentLibJar));
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
workflow.setConf(newConf.toXmlString());
Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
fs.create(subwfLibJar);
assertTrue(fs.exists(subwfLibJar));
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
childConf = wps.createProtoActionConf(childConf, "authToken", true);
assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
}
protected WorkflowActionBean createWorkflowActionSetPending(
String wfId, WorkflowAction.Status status) throws Exception {
WorkflowActionBean action = new WorkflowActionBean();
String actionname = "testAction";
action.setName(actionname);
action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname));
action.setJobId(wfId);
action.setType("map-reduce");
action.setTransition("transition");
action.setStatus(status);
action.setStartTime(new Date());
action.setEndTime(new Date());
action.setLastCheckTime(new Date());
action.setPending();
action.setExecutionPath("/");
Path inputDir = new Path(getFsTestCaseDir(), "input");
Path outputDir = new Path(getFsTestCaseDir(), "output");
FileSystem fs = getFileSystem();
Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
w.write("dummy\n");
w.write("dummy\n");
w.close();
String actionXml =
"<map-reduce>"
+ "<job-tracker>"
+ getJobTrackerUri()
+ "</job-tracker>"
+ "<name-node>"
+ getNameNodeUri()
+ "</name-node>"
+ "<prepare><delete path=\""
+ outputDir.toString()
+ "\"/></prepare>"
+ "<configuration>"
+ "<property><name>mapred.mapper.class</name><value>"
+ MapperReducerForTest.class.getName()
+ "</value></property>"
+ "<property><name>mapred.reducer.class</name><value>"
+ MapperReducerForTest.class.getName()
+ "</value></property>"
+ "<property><name>mapred.input.dir</name><value>"
+ inputDir.toString()
+ "</value></property>"
+ "<property><name>mapred.output.dir</name><value>"
+ outputDir.toString()
+ "</value></property>"
+ "</configuration>"
+ "</map-reduce>";
action.setConf(actionXml);
return action;
}
private Context createContext(String actionXml) throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
SharelibUtils.addToDistributedCache("hive2", getFileSystem(), getFsTestCaseDir(), protoConf);
WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
action.setType(ae.getType());
action.setConf(actionXml);
return new Context(wf, action);
}
public void testGetGroupFromParent() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
String actionConf =
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>";
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(actionConf);
// negative test
final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
workflow.setConf(newConf.toXmlString());
subWorkflow.start(new Context(workflow, action), action);
OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
5000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
subWorkflow.check(new Context(workflow, action), action);
return action.getStatus() == WorkflowActionBean.Status.DONE;
}
});
subWorkflow.check(new Context(workflow, action), action);
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME));
// positive test
newConf.set(OozieClient.GROUP_NAME, getTestGroup());
workflow.setConf(newConf.toXmlString());
final WorkflowActionBean action1 = new WorkflowActionBean();
action1.setConf(actionConf);
action1.setId("W1");
subWorkflow.start(new Context(workflow, action1), action1);
oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL);
waitFor(
5000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
subWorkflow.check(new Context(workflow, action1), action1);
return action1.getStatus() == WorkflowActionBean.Status.DONE;
}
});
subWorkflow.check(new Context(workflow, action1), action1);
subWorkflow.end(new Context(workflow, action1), action1);
wf = oozieClient.getJobInfo(action1.getExternalId());
childConf = new XConfiguration(new StringReader(wf.getConf()));
assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME));
}
public void testSubWorkflowRecovery() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
String extId = action.getExternalId();
assertEquals(WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(extId).getStatus());
WorkflowActionBean action1 = new WorkflowActionBean();
action1.setId(action.getId());
action1.setName(action.getName());
action1.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
+ " <app-path>wrongAppPath</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
subWorkflow.start(new Context(workflow, action1), action1);
assertEquals(extId, action1.getExternalId());
subWorkflow.check(new Context(workflow, action1), action1);
assertEquals(WorkflowAction.Status.DONE, action1.getStatus());
subWorkflow.end(new Context(workflow, action1), action1);
assertEquals(WorkflowAction.Status.OK, action1.getStatus());
}
/**
* Provides functionality to test non transient failures.
*
* @param errorType the error type. (start.non-transient, end.non-transient)
* @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
* @param expErrorMsg expected error message.
* @throws Exception
*/
private void _testNonTransient(
String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
String workflowPath = getTestCaseFileUri("workflow.xml");
Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
IOUtils.copyCharStream(reader, writer);
final DagEngine engine = new DagEngine("u");
Configuration conf = new XConfiguration();
conf.set(OozieClient.APP_PATH, workflowPath);
conf.set(OozieClient.USER_NAME, getTestUser());
conf.set(OozieClient.LOG_TOKEN, "t");
conf.set("signal-value", "OK");
conf.set("external-status", "ok");
conf.set("error", errorType);
final String jobId = engine.submitJob(conf, true);
waitFor(
5000,
new Predicate() {
public boolean evaluate() throws Exception {
return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
}
});
final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
store.beginTrx();
List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, true);
int n = actions.size();
WorkflowActionBean action = actions.get(n - 1);
assertEquals("TEST_ERROR", action.getErrorCode());
assertEquals(expErrorMsg, action.getErrorMessage());
assertEquals(expStatus1, action.getStatus());
assertTrue(action.isPending() == false);
assertTrue(engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
String actionConf = action.getConf();
String fixedActionConf = actionConf.replaceAll(errorType, "none");
action.setConf(fixedActionConf);
store.updateAction(action);
store.commitTrx();
store.closeTrx();
engine.resume(jobId);
waitFor(
5000,
new Predicate() {
public boolean evaluate() throws Exception {
return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED);
}
});
assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());
final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
store2.beginTrx();
actions = store2.getActionsForWorkflow(jobId, false);
action = actions.get(0);
assertEquals(null, action.getErrorCode());
assertEquals(null, action.getErrorMessage());
assertEquals(WorkflowActionBean.Status.OK, action.getStatus());
store2.commitTrx();
store2.closeTrx();
}