private RunningJob submitAction(Context context, Namespace ns) throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
WorkflowAction action = context.getAction();
ae.prepareActionDir(getFileSystem(), context);
ae.submitLauncher(getFileSystem(), context, action);
String jobId = action.getExternalId();
String jobTracker = action.getTrackerUri();
String consoleUrl = action.getConsoleUrl();
assertNotNull(jobId);
assertNotNull(jobTracker);
assertNotNull(consoleUrl);
Element e = XmlUtils.parseXml(action.getConf());
XConfiguration conf =
new XConfiguration(
new StringReader(XmlUtils.prettyPrint(e.getChild("configuration", ns)).toString()));
conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker", ns));
conf.set("fs.default.name", e.getChildTextTrim("name-node", ns));
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf);
final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
assertNotNull(runningJob);
return runningJob;
}
@SuppressWarnings("unchecked")
public void testSetupMethodsForQuery() throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
List<Class> classes = new ArrayList<Class>();
classes.add(Hive2Main.class);
assertEquals(classes, ae.getLauncherClasses());
String sampleQuery = "SELECT count(*) from foobar";
Element actionXml =
XmlUtils.parseXml(
"<hive2 xmlns=\"uri:oozie:hive2-action:0.2\">"
+ "<job-tracker>"
+ getJobTrackerUri()
+ "</job-tracker>"
+ "<name-node>"
+ getNameNodeUri()
+ "</name-node>"
+ "<jdbc-url>jdbc:hive2://foo:1234/bar</jdbc-url>"
+ "<password>pass</password>"
+ "<query>"
+ sampleQuery
+ "</query>"
+ "<param>a=A</param>"
+ "<param>b=B</param>"
+ "<argument>-c</argument>"
+ "<argument>--dee</argument>"
+ "</hive2>");
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
action.setType(ae.getType());
Context context = new Context(wf, action);
Configuration conf = ae.createBaseHadoopConf(context, actionXml);
ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir());
assertEquals("jdbc:hive2://foo:1234/bar", conf.get("oozie.hive2.jdbc.url"));
assertEquals("pass", conf.get("oozie.hive2.password"));
assertEquals(sampleQuery, conf.get("oozie.hive2.query"));
assertNull(conf.get("oozie.hive2.script"));
assertEquals("2", conf.get("oozie.hive2.params.size"));
assertEquals("a=A", conf.get("oozie.hive2.params.0"));
assertEquals("b=B", conf.get("oozie.hive2.params.1"));
assertEquals("2", conf.get("oozie.hive2.args.size"));
assertEquals("-c", conf.get("oozie.hive2.args.0"));
assertEquals("--dee", conf.get("oozie.hive2.args.1"));
}
private Context createContext(String actionXml) throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
SharelibUtils.addToDistributedCache("hive2", getFileSystem(), getFsTestCaseDir(), protoConf);
WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
action.setType(ae.getType());
action.setConf(actionXml);
return new Context(wf, action);
}
@SuppressWarnings("deprecation")
public void testHive2Action() throws Exception {
setupHiveServer2();
Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME);
Path outputDir = new Path(getFsTestCaseDir(), OUTPUT_DIRNAME);
FileSystem fs = getFileSystem();
{
String query = getHive2Script(inputDir.toString(), outputDir.toString());
Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
dataWriter.write(SAMPLE_DATA_TEXT);
dataWriter.close();
Context context = createContext(getQueryActionXml(query));
final RunningJob launcherJob =
submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2"));
String launcherId = context.getAction().getExternalId();
waitFor(
200 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return launcherJob.isComplete();
}
});
assertTrue(launcherJob.isSuccessful());
Configuration conf = new XConfiguration();
conf.set("user.name", getTestUser());
Map<String, String> actionData =
LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
Hive2ActionExecutor ae = new Hive2ActionExecutor();
ae.check(context, context.getAction());
assertTrue(launcherId.equals(context.getAction().getExternalId()));
assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
ae.end(context, context.getAction());
assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
// Disabled external child id check until Hive version is upgraded to 0.14+
// assertNotNull(context.getExternalChildIDs());
assertTrue(fs.exists(outputDir));
assertTrue(fs.isDirectory(outputDir));
}
{
Path script = new Path(getAppPath(), HIVE_SCRIPT_FILENAME);
Writer scriptWriter = new OutputStreamWriter(fs.create(script));
scriptWriter.write(getHive2Script(inputDir.toString(), outputDir.toString()));
scriptWriter.close();
Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
dataWriter.write(SAMPLE_DATA_TEXT);
dataWriter.close();
Context context = createContext(getScriptActionXml());
final RunningJob launcherJob =
submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.1"));
String launcherId = context.getAction().getExternalId();
waitFor(
200 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return launcherJob.isComplete();
}
});
assertTrue(launcherJob.isSuccessful());
Configuration conf = new XConfiguration();
conf.set("user.name", getTestUser());
Map<String, String> actionData =
LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
Hive2ActionExecutor ae = new Hive2ActionExecutor();
ae.check(context, context.getAction());
assertTrue(launcherId.equals(context.getAction().getExternalId()));
assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
ae.end(context, context.getAction());
assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
// Disabled external child id check until Hive version is upgraded to 0.14+
// assertNotNull(context.getExternalChildIDs());
assertTrue(fs.exists(outputDir));
assertTrue(fs.isDirectory(outputDir));
}
// Negative testcase with incorrect hive-query.
{
String query = getHive2BadScript(inputDir.toString(), outputDir.toString());
Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
dataWriter.write(SAMPLE_DATA_TEXT);
dataWriter.close();
Context context = createContext(getQueryActionXml(query));
final RunningJob launcherJob =
submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2"));
String launcherId = context.getAction().getExternalId();
waitFor(
200 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return launcherJob.isComplete();
}
});
assertTrue(launcherJob.isSuccessful());
Configuration conf = new XConfiguration();
conf.set("user.name", getTestUser());
Map<String, String> actionData =
LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
Hive2ActionExecutor ae = new Hive2ActionExecutor();
ae.check(context, context.getAction());
assertTrue(launcherId.equals(context.getAction().getExternalId()));
assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
ae.end(context, context.getAction());
assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
assertNull(context.getExternalChildIDs());
}
}