public static WorkflowJob.Status killedOozieJob(String oozieJobid) throws OozieClientException {
OozieClient wc = new AuthOozieClient("http://10.25.22.19:8080/oozie", "ldap");
wc.setHeader("j_username", "V_PA011_HADOOP_CORE");
wc.setHeader("j_password", "{DES}/+J5e0Ed2/PABTqzeVGQ1g==");
wc.kill(oozieJobid);
return wc.getJobInfo(oozieJobid).getStatus();
}
public void testConfigNotPropagation() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set("abc", "xyz");
workflow.setConf(newConf.toXmlString());
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
assertNull(childConf.get("abc"));
}
/**
* Run recipe with different frequencies. Submission should go through. Check frequency of the
* launched oozie job
*/
@Test(dataProvider = "frequencyGenerator")
public void differentRecipeFrequenciesTest(String frequency) throws Exception {
setUp(RecipeExecLocation.SourceCluster);
LOGGER.info("Testing with frequency: " + frequency);
String tblName = "myTable";
recipeMerlin
.withSourceDb(DB_NAME)
.withSourceTable(tblName)
.withFrequency(new Frequency(frequency));
runSql(connection, "create table " + tblName + "(comment string)");
final List<String> command = recipeMerlin.getSubmissionCommand();
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
LOGGER.info("Submission went through.");
InstanceUtil.waitTillInstanceReachState(
clusterOC, recipeMerlin.getName(), 1, CoordinatorAction.Status.RUNNING, EntityType.PROCESS);
String filter = "name=FALCON_PROCESS_" + recipeMerlin.getName();
List<BundleJob> bundleJobs = OozieUtil.getBundles(clusterOC, filter, 0, 10);
List<String> bundleIds = OozieUtil.getBundleIds(bundleJobs);
String bundleId = OozieUtil.getMaxId(bundleIds);
List<CoordinatorJob> coords = clusterOC.getBundleJobInfo(bundleId).getCoordinators();
List<String> cIds = new ArrayList<String>();
for (CoordinatorJob coord : coords) {
cIds.add(coord.getId());
}
String coordId = OozieUtil.getMinId(cIds);
CoordinatorJob job = clusterOC.getCoordJobInfo(coordId);
CoordinatorJob.Timeunit timeUnit = job.getTimeUnit();
String freq = job.getFrequency();
LOGGER.info("Frequency of running job: " + timeUnit + " " + freq);
Assert.assertTrue(
frequency.contains(timeUnit.name().toLowerCase().replace("_", ""))
&& frequency.contains(freq),
"Running job has different frequency.");
}
public void testSubworkflowLib() throws Exception {
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
FileSystem fs = getFileSystem();
Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
fs.create(parentLibJar);
assertTrue(fs.exists(parentLibJar));
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
workflow.setConf(newConf.toXmlString());
Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
fs.create(subwfLibJar);
assertTrue(fs.exists(subwfLibJar));
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
childConf = wps.createProtoActionConf(childConf, "authToken", true);
assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
}
private void copyOozieLog(OozieClient client, FileSystem fs, Path path, String id)
throws OozieClientException, IOException {
InputStream in = new ByteArrayInputStream(client.getJobLog(id).getBytes());
OutputStream out = fs.create(new Path(path, "oozie.log"));
IOUtils.copyBytes(in, out, 4096, true);
LOG.info("Copied oozie log to {}", path);
}
/*
* Retrieves replication coordinator instances.
* @param client target oozie client
* @param fName feed name
*/
private List<CoordinatorAction> getReplicationInstances(OozieClient client, String fName)
throws OozieClientException {
String filter = "name=FALCON_FEED_" + fName;
List<BundleJob> bundleJobs = OozieUtil.getBundles(client, filter, 0, 10);
Assert.assertNotEquals(bundleJobs.size(), 0, "Could not retrieve bundles");
List<String> bundleIds = OozieUtil.getBundleIds(bundleJobs);
String bundleId = OozieUtil.getMaxId(bundleIds);
LOGGER.info(String.format("Using bundle %s", bundleId));
List<CoordinatorJob> coords = client.getBundleJobInfo(bundleId).getCoordinators();
String coordId = null;
for (CoordinatorJob coord : coords) {
if (coord.getAppName().contains("FEED_REPLICATION")) {
coordId = coord.getId();
break;
}
}
LOGGER.info(String.format("Using coordinator id: %s", coordId));
Assert.assertNotNull(coordId, "Replication coordinator not found.");
CoordinatorJob coordinatorJob = client.getCoordJobInfo(coordId);
return coordinatorJob.getActions();
}
@Test
public void testGetValidationWarnings_incompatibleVersions() throws Exception {
OozieJobExecutorConfig config = new OozieJobExecutorConfig();
OozieClient client = getBadConfigTest();
OozieJobExecutorJobEntry je =
new TestOozieJobExecutorJobEntry(client); // just use this to force an error condition
config.setOozieUrl("http://localhost/oozie");
config.setOozieWorkflowConfig("file:///test/job.properties");
config.setJobEntryName("name");
List<String> warnings = je.getValidationWarnings(config);
assertEquals(1, warnings.size());
assertEquals(
BaseMessages.getString(
OozieJobExecutorJobEntry.class,
"ValidationMessages.Incompatible.Oozie.Versions",
client.getClientBuildVersion()),
warnings.get(0));
}
public static WorkflowJob.Status queryJobStatus(String oozieJobid)
throws OozieClientException, InterruptedException {
OozieClient wc = new AuthOozieClient("http://10.25.22.19:8080/oozie", "ldap");
wc.setHeader("j_username", "V_PA011_HADOOP_CORE");
wc.setHeader("j_password", "{DES}/+J5e0Ed2/PABTqzeVGQ1g==");
List<WorkflowAction> walist;
int wacount = 0;
while (wacount < 2) {
walist = wc.getJobInfo(oozieJobid).getActions();
wacount = walist.size();
Thread.sleep(20000);
System.out.println("#######-" + wacount);
for (WorkflowAction wa : walist) {
System.out.println("#######" + wa.getName());
if (wa.getConsoleUrl().contains("jobdetails.jsp?jobid=")) {
System.out.println("#######" + wa.getConsoleUrl());
}
}
}
return wc.getJobInfo(oozieJobid).getStatus();
}
public static String oozieSubmitJob() throws Exception {
return OozieClient.doAs(
"hadoop",
new Callable<String>() {
@Override
public String call() throws Exception {
OozieClient wc = new AuthOozieClient("http://10.25.22.19:8080/oozie", "ldap");
Properties conf = wc.createConfiguration();
// setting workflow parameters
// wc.doAs("hadoop");
wc.setHeader("j_username", "V_PA011_HADOOP_CORE");
wc.setHeader("j_password", "{DES}/+J5e0Ed2/PABTqzeVGQ1g==");
conf.setProperty(OozieClient.USER_NAME, "hadoop");
conf.setProperty(OozieClient.GROUP_NAME, "hadoop");
conf.setProperty(
OozieClient.APP_PATH, "hdfs://10.25.22.19:49000/apps/actionTest/shellaction");
conf.setProperty("namenode_address", "hdfs://10.25.22.19:49000");
conf.setProperty("jobtracker_address", "10.25.22.19:49001");
conf.setProperty("mapred_job_queue_name", "queue01");
conf.setProperty("data_db_username", "PSCPHDP");
conf.setProperty("data_db_password", "{DES}4ueyVO2J9+v8ML7FGs608g==");
conf.setProperty("data_db_url", "jdbc:oracle:thin:@10.25.18.28:1582:pa18dvp");
conf.setProperty("nominalformateDate", "2013-04-12");
conf.setProperty("oozie.use.system.libpath", "true");
String jobId = wc.run(conf);
System.out.println(jobId);
// wait until the workflo w job finishes printing the status every 10 seconds
// while (wc.getJobInfo(jobId).getStatus() == WorkflowJob.Status.RUNNING) {
// System.out.println("Workflow job running ...");
// Thread.sleep(10 * 1000);
// }
// print the final status o the workflow job
System.out.println("Workflow job completed ...");
System.out.println(wc.getJobInfo(jobId));
return jobId;
}
});
}
public void testFsFailover() throws Exception {
Reader reader = IOUtils.getResourceAsReader("failover-fs-wf.xml", -1);
Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
IOUtils.copyCharStream(reader, writer);
final OozieClient wfClient = LocalOozie.getClient();
Properties conf = wfClient.createConfiguration();
conf.setProperty(OozieClient.APP_PATH, getTestCaseDir());
conf.setProperty(OozieClient.USER_NAME, getTestUser());
conf.setProperty(OozieClient.GROUP_NAME, getTestGroup());
injectKerberosInfo(conf);
final Path source = new Path(getFsTestCaseDir(), "fsfailover-source");
getFileSystem().mkdirs(source);
final Path target = new Path(getFsTestCaseDir().toString(), "fsfailover-target");
conf.setProperty("source", source.toString());
conf.setProperty("target", target.toUri().getPath());
setSystemProperty(FaultInjection.FAULT_INJECTION, "true");
setSystemProperty(SkipCommitFaultInjection.ACTION_FAILOVER_FAULT_INJECTION, "true");
final String jobId1 = wfClient.submit(conf);
wfClient.start(jobId1);
waitFor(
10 * 1000,
new Predicate() {
public boolean evaluate() throws Exception {
return getFileSystem().exists(target);
}
});
assertTrue(getFileSystem().exists(target));
waitFor(
10 * 1000,
new Predicate() {
public boolean evaluate() throws Exception {
return FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection");
}
});
assertFalse(FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection"));
assertEquals(WorkflowJob.Status.RUNNING, wfClient.getJobInfo(jobId1).getStatus());
WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId1, false);
WorkflowActionBean action = actions.get(0);
assertEquals(WorkflowAction.Status.PREP, action.getStatus());
setSystemProperty(FaultInjection.FAULT_INJECTION, "false");
setSystemProperty(SkipCommitFaultInjection.ACTION_FAILOVER_FAULT_INJECTION, "false");
ActionStartCommand actionStartCommand =
new ActionStartCommand(action.getId(), action.getType());
actionStartCommand.call();
store = Services.get().get(WorkflowStoreService.class).create();
actions = store.getActionsForWorkflow(jobId1, false);
action = actions.get(0);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
waitFor(
5 * 1000,
new Predicate() {
public boolean evaluate() throws Exception {
return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(WorkflowJob.Status.SUCCEEDED, wfClient.getJobInfo(jobId1).getStatus());
final String jobId2 = wfClient.submit(conf);
wfClient.start(jobId2);
waitFor(
10 * 1000,
new Predicate() {
public boolean evaluate() throws Exception {
return wfClient.getJobInfo(jobId2).getStatus() == WorkflowJob.Status.KILLED;
}
});
assertEquals(WorkflowJob.Status.KILLED, wfClient.getJobInfo(jobId2).getStatus());
}
public void testGetGroupFromParent() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
String actionConf =
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>";
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(actionConf);
// negative test
final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
workflow.setConf(newConf.toXmlString());
subWorkflow.start(new Context(workflow, action), action);
OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
5000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
subWorkflow.check(new Context(workflow, action), action);
return action.getStatus() == WorkflowActionBean.Status.DONE;
}
});
subWorkflow.check(new Context(workflow, action), action);
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME));
// positive test
newConf.set(OozieClient.GROUP_NAME, getTestGroup());
workflow.setConf(newConf.toXmlString());
final WorkflowActionBean action1 = new WorkflowActionBean();
action1.setConf(actionConf);
action1.setId("W1");
subWorkflow.start(new Context(workflow, action1), action1);
oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL);
waitFor(
5000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
subWorkflow.check(new Context(workflow, action1), action1);
return action1.getStatus() == WorkflowActionBean.Status.DONE;
}
});
subWorkflow.check(new Context(workflow, action1), action1);
subWorkflow.end(new Context(workflow, action1), action1);
wf = oozieClient.getJobInfo(action1.getExternalId());
childConf = new XConfiguration(new StringReader(wf.getConf()));
assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME));
}
public void testSubWorkflowRecovery() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
String extId = action.getExternalId();
assertEquals(WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(extId).getStatus());
WorkflowActionBean action1 = new WorkflowActionBean();
action1.setId(action.getId());
action1.setName(action.getName());
action1.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
+ " <app-path>wrongAppPath</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
subWorkflow.start(new Context(workflow, action1), action1);
assertEquals(extId, action1.getExternalId());
subWorkflow.check(new Context(workflow, action1), action1);
assertEquals(WorkflowAction.Status.DONE, action1.getStatus());
subWorkflow.end(new Context(workflow, action1), action1);
assertEquals(WorkflowAction.Status.OK, action1.getStatus());
}
public int run(WorkflowExecutionContext context) {
try {
OozieClient client = new OozieClient(context.getWorkflowEngineUrl());
WorkflowJob jobInfo;
try {
jobInfo = client.getJobInfo(context.getUserSubflowId());
} catch (OozieClientException e) {
LOG.error("Error getting jobinfo for: {}", context.getUserSubflowId(), e);
return 0;
}
// Assumption is - Each wf run will have a directory
// the corresponding job logs are stored within the respective dir
Path path =
new Path(context.getLogDir() + "/" + String.format("%03d", context.getWorkflowRunId()));
FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri(), getConf());
if (EntityType.FEED.name().equalsIgnoreCase(context.getEntityType())
|| notUserWorkflowEngineIsOozie(context.getUserWorkflowEngine())) {
// if replication wf, retention wf or PIG Process
copyOozieLog(client, fs, path, jobInfo.getId());
List<WorkflowAction> workflowActions = jobInfo.getActions();
for (int i = 0; i < workflowActions.size(); i++) {
if (FALCON_ACTIONS.contains(workflowActions.get(i).getName())) {
copyTTlogs(fs, path, jobInfo.getActions().get(i));
break;
}
}
} else {
String flowId;
// if process wf with pig, hive
if (context.getUserWorkflowEngine().equals("pig")
|| context.getUserWorkflowEngine().equals("hive")) {
flowId = jobInfo.getId();
} else {
// if process wf with oozie engine
flowId = jobInfo.getExternalId();
}
copyOozieLog(client, fs, path, flowId);
WorkflowJob subflowInfo = client.getJobInfo(flowId);
List<WorkflowAction> actions = subflowInfo.getActions();
for (WorkflowAction action : actions) {
if (isActionTypeSupported(action)) {
LOG.info(
"Copying hadoop TT log for action: {} of type: {}",
action.getName(),
action.getType());
copyTTlogs(fs, path, action);
} else {
LOG.info(
"Ignoring hadoop TT log for non supported action: {} of type: {}",
action.getName(),
action.getType());
}
}
}
} catch (Exception e) {
// JobLogMover doesn't throw exception, a failed log mover will not fail the user workflow
LOG.error("Exception in log mover:", e);
}
return 0;
}