private RunningJob submitAction(Context context, Namespace ns) throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); WorkflowAction action = context.getAction(); ae.prepareActionDir(getFileSystem(), context); ae.submitLauncher(getFileSystem(), context, action); String jobId = action.getExternalId(); String jobTracker = action.getTrackerUri(); String consoleUrl = action.getConsoleUrl(); assertNotNull(jobId); assertNotNull(jobTracker); assertNotNull(consoleUrl); Element e = XmlUtils.parseXml(action.getConf()); XConfiguration conf = new XConfiguration( new StringReader(XmlUtils.prettyPrint(e.getChild("configuration", ns)).toString())); conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker", ns)); conf.set("fs.default.name", e.getChildTextTrim("name-node", ns)); conf.set("user.name", context.getProtoActionConf().get("user.name")); conf.set("group.name", getTestGroup()); JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker); XConfiguration.copy(conf, jobConf); String user = jobConf.get("user.name"); JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf); final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId)); assertNotNull(runningJob); return runningJob; }
@SuppressWarnings("unchecked") public void testSetupMethodsForQuery() throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); List<Class> classes = new ArrayList<Class>(); classes.add(Hive2Main.class); assertEquals(classes, ae.getLauncherClasses()); String sampleQuery = "SELECT count(*) from foobar"; Element actionXml = XmlUtils.parseXml( "<hive2 xmlns=\"uri:oozie:hive2-action:0.2\">" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<jdbc-url>jdbc:hive2://foo:1234/bar</jdbc-url>" + "<password>pass</password>" + "<query>" + sampleQuery + "</query>" + "<param>a=A</param>" + "<param>b=B</param>" + "<argument>-c</argument>" + "<argument>--dee</argument>" + "</hive2>"); XConfiguration protoConf = new XConfiguration(); protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action"); WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); action.setType(ae.getType()); Context context = new Context(wf, action); Configuration conf = ae.createBaseHadoopConf(context, actionXml); ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir()); assertEquals("jdbc:hive2://foo:1234/bar", conf.get("oozie.hive2.jdbc.url")); assertEquals("pass", conf.get("oozie.hive2.password")); assertEquals(sampleQuery, conf.get("oozie.hive2.query")); assertNull(conf.get("oozie.hive2.script")); assertEquals("2", conf.get("oozie.hive2.params.size")); assertEquals("a=A", conf.get("oozie.hive2.params.0")); assertEquals("b=B", conf.get("oozie.hive2.params.1")); assertEquals("2", conf.get("oozie.hive2.args.size")); assertEquals("-c", conf.get("oozie.hive2.args.0")); assertEquals("--dee", conf.get("oozie.hive2.args.1")); }
private Context createContext(String actionXml) throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); XConfiguration protoConf = new XConfiguration(); protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); SharelibUtils.addToDistributedCache("hive2", getFileSystem(), getFsTestCaseDir(), protoConf); WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action"); WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); action.setType(ae.getType()); action.setConf(actionXml); return new Context(wf, action); }
@SuppressWarnings("deprecation") public void testHive2Action() throws Exception { setupHiveServer2(); Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME); Path outputDir = new Path(getFsTestCaseDir(), OUTPUT_DIRNAME); FileSystem fs = getFileSystem(); { String query = getHive2Script(inputDir.toString(), outputDir.toString()); Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME))); dataWriter.write(SAMPLE_DATA_TEXT); dataWriter.close(); Context context = createContext(getQueryActionXml(query)); final RunningJob launcherJob = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2")); String launcherId = context.getAction().getExternalId(); waitFor( 200 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { return launcherJob.isComplete(); } }); assertTrue(launcherJob.isSuccessful()); Configuration conf = new XConfiguration(); conf.set("user.name", getTestUser()); Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf); assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); Hive2ActionExecutor ae = new Hive2ActionExecutor(); ae.check(context, context.getAction()); assertTrue(launcherId.equals(context.getAction().getExternalId())); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); // Disabled external child id check until Hive version is upgraded to 0.14+ // assertNotNull(context.getExternalChildIDs()); assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } { Path script = new Path(getAppPath(), HIVE_SCRIPT_FILENAME); Writer scriptWriter = new OutputStreamWriter(fs.create(script)); scriptWriter.write(getHive2Script(inputDir.toString(), outputDir.toString())); scriptWriter.close(); Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME))); dataWriter.write(SAMPLE_DATA_TEXT); dataWriter.close(); Context context = createContext(getScriptActionXml()); final RunningJob launcherJob = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.1")); String launcherId = context.getAction().getExternalId(); waitFor( 200 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { return launcherJob.isComplete(); } }); assertTrue(launcherJob.isSuccessful()); Configuration conf = new XConfiguration(); conf.set("user.name", getTestUser()); Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf); assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); Hive2ActionExecutor ae = new Hive2ActionExecutor(); ae.check(context, context.getAction()); assertTrue(launcherId.equals(context.getAction().getExternalId())); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); // Disabled external child id check until Hive version is upgraded to 0.14+ // assertNotNull(context.getExternalChildIDs()); assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } // Negative testcase with incorrect hive-query. { String query = getHive2BadScript(inputDir.toString(), outputDir.toString()); Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME))); dataWriter.write(SAMPLE_DATA_TEXT); dataWriter.close(); Context context = createContext(getQueryActionXml(query)); final RunningJob launcherJob = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2")); String launcherId = context.getAction().getExternalId(); waitFor( 200 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { return launcherJob.isComplete(); } }); assertTrue(launcherJob.isSuccessful()); Configuration conf = new XConfiguration(); conf.set("user.name", getTestUser()); Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf); assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); Hive2ActionExecutor ae = new Hive2ActionExecutor(); ae.check(context, context.getAction()); assertTrue(launcherId.equals(context.getAction().getExternalId())); assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); assertNull(context.getExternalChildIDs()); } }