private RunningJob submitAction(Context context, Namespace ns) throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
WorkflowAction action = context.getAction();
ae.prepareActionDir(getFileSystem(), context);
ae.submitLauncher(getFileSystem(), context, action);
String jobId = action.getExternalId();
String jobTracker = action.getTrackerUri();
String consoleUrl = action.getConsoleUrl();
assertNotNull(jobId);
assertNotNull(jobTracker);
assertNotNull(consoleUrl);
Element e = XmlUtils.parseXml(action.getConf());
XConfiguration conf =
new XConfiguration(
new StringReader(XmlUtils.prettyPrint(e.getChild("configuration", ns)).toString()));
conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker", ns));
conf.set("fs.default.name", e.getChildTextTrim("name-node", ns));
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf);
final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
assertNotNull(runningJob);
return runningJob;
}
public void testConfigNotPropagation() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set("abc", "xyz");
workflow.setConf(newConf.toXmlString());
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
assertNull(childConf.get("abc"));
}
/**
* Returns a new XConfiguration instance with all inline values resolved.
*
* @return a new XConfiguration instance with all inline values resolved.
*/
public XConfiguration resolve() {
XConfiguration resolved = new XConfiguration();
for (Map.Entry<String, String> entry : this) {
resolved.set(entry.getKey(), get(entry.getKey()));
}
return resolved;
}
/**
* Returns a new XConfiguration with all values trimmed.
*
* @return a new XConfiguration with all values trimmed.
*/
public XConfiguration trim() {
XConfiguration trimmed = new XConfiguration();
for (Map.Entry<String, String> entry : this) {
trimmed.set(entry.getKey(), entry.getValue().trim());
}
return trimmed;
}
public void testSubworkflowLib() throws Exception {
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
FileSystem fs = getFileSystem();
Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
fs.create(parentLibJar);
assertTrue(fs.exists(parentLibJar));
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
workflow.setConf(newConf.toXmlString());
Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
fs.create(subwfLibJar);
assertTrue(fs.exists(subwfLibJar));
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
childConf = wps.createProtoActionConf(childConf, "authToken", true);
assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
}
/**
* Return a configuration with all sensitive values masked.
*
* @return masked configuration.
*/
public Configuration getMaskedConfiguration() {
XConfiguration maskedConf = new XConfiguration();
Configuration conf = getConf();
for (Map.Entry<String, String> entry : conf) {
String name = entry.getKey();
String value = getValue(conf, name);
maskedConf.set(name, value);
}
return maskedConf;
}
@SuppressWarnings("unchecked")
public void testSetupMethodsForQuery() throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
List<Class> classes = new ArrayList<Class>();
classes.add(Hive2Main.class);
assertEquals(classes, ae.getLauncherClasses());
String sampleQuery = "SELECT count(*) from foobar";
Element actionXml =
XmlUtils.parseXml(
"<hive2 xmlns=\"uri:oozie:hive2-action:0.2\">"
+ "<job-tracker>"
+ getJobTrackerUri()
+ "</job-tracker>"
+ "<name-node>"
+ getNameNodeUri()
+ "</name-node>"
+ "<jdbc-url>jdbc:hive2://foo:1234/bar</jdbc-url>"
+ "<password>pass</password>"
+ "<query>"
+ sampleQuery
+ "</query>"
+ "<param>a=A</param>"
+ "<param>b=B</param>"
+ "<argument>-c</argument>"
+ "<argument>--dee</argument>"
+ "</hive2>");
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
action.setType(ae.getType());
Context context = new Context(wf, action);
Configuration conf = ae.createBaseHadoopConf(context, actionXml);
ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir());
assertEquals("jdbc:hive2://foo:1234/bar", conf.get("oozie.hive2.jdbc.url"));
assertEquals("pass", conf.get("oozie.hive2.password"));
assertEquals(sampleQuery, conf.get("oozie.hive2.query"));
assertNull(conf.get("oozie.hive2.script"));
assertEquals("2", conf.get("oozie.hive2.params.size"));
assertEquals("a=A", conf.get("oozie.hive2.params.0"));
assertEquals("b=B", conf.get("oozie.hive2.params.1"));
assertEquals("2", conf.get("oozie.hive2.args.size"));
assertEquals("-c", conf.get("oozie.hive2.args.0"));
assertEquals("--dee", conf.get("oozie.hive2.args.1"));
}
private Context createContext(String actionXml) throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
SharelibUtils.addToDistributedCache("hive2", getFileSystem(), getFsTestCaseDir(), protoConf);
WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
action.setType(ae.getType());
action.setConf(actionXml);
return new Context(wf, action);
}
/**
* Merge default configuration with user-defined configuration.
*
* @throws CommandException thrown if failed to read or merge configurations
*/
protected void mergeDefaultConfig() throws CommandException {
Path configDefault = null;
try {
String coordAppPathStr = conf.get(OozieClient.COORDINATOR_APP_PATH);
Path coordAppPath = new Path(coordAppPathStr);
String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
String group =
ParamChecker.notEmpty(conf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
FileSystem fs =
Services.get()
.get(HadoopAccessorService.class)
.createFileSystem(user, group, coordAppPath.toUri(), new Configuration());
// app path could be a directory
if (!fs.isFile(coordAppPath)) {
configDefault = new Path(coordAppPath, CONFIG_DEFAULT);
} else {
configDefault = new Path(coordAppPath.getParent(), CONFIG_DEFAULT);
}
if (fs.exists(configDefault)) {
Configuration defaultConf = new XConfiguration(fs.open(configDefault));
PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
XConfiguration.injectDefaults(defaultConf, conf);
} else {
LOG.info("configDefault Doesn't exist " + configDefault);
}
PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
// Resolving all variables in the job properties.
// This ensures the Hadoop Configuration semantics is preserved.
XConfiguration resolvedVarsConf = new XConfiguration();
for (Map.Entry<String, String> entry : conf) {
resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
}
conf = resolvedVarsConf;
} catch (IOException e) {
throw new CommandException(
ErrorCode.E0702,
e.getMessage() + " : Problem reading default config " + configDefault,
e);
} catch (HadoopAccessorException e) {
throw new CommandException(e);
}
LOG.debug("Merged CONF :" + XmlUtils.prettyPrint(conf).toString());
}
private void setValue(String name, String value) {
super.set(name, value);
}
private LogChangesConfiguration loadConf() throws ServiceException {
XConfiguration configuration;
try {
InputStream inputStream = getDefaultConfiguration();
configuration = loadConfig(inputStream, true);
File file = new File(configFile);
if (!file.exists()) {
log.info("Missing site configuration file [{0}]", configFile);
} else {
inputStream = new FileInputStream(configFile);
XConfiguration siteConfiguration = loadConfig(inputStream, false);
XConfiguration.injectDefaults(configuration, siteConfiguration);
configuration = siteConfiguration;
}
} catch (IOException ex) {
throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
}
if (log.isTraceEnabled()) {
try {
StringWriter writer = new StringWriter();
for (Map.Entry<String, String> entry : configuration) {
String value = getValue(configuration, entry.getKey());
writer.write(" " + entry.getKey() + " = " + value + "\n");
}
writer.close();
log.trace("Configuration:\n{0}---", writer.toString());
} catch (IOException ex) {
throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
}
}
String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
if (ignoreSysProps != null) {
IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
}
for (Map.Entry<String, String> entry : configuration) {
String sysValue = System.getProperty(entry.getKey());
if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
configuration.set(entry.getKey(), sysValue);
}
}
for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
String name = (String) entry.getKey();
if (!IGNORE_SYS_PROPS.contains(name)) {
if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
if (configuration.get(name) == null) {
log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
}
}
}
}
// Backward compatible, we should still support -Dparam.
for (String key : CONF_SYS_PROPS) {
String sysValue = System.getProperty(key);
if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) {
log.info(
"Overriding configuration with system property. Key [{0}], Value [{1}] ",
key, sysValue);
configuration.set(key, sysValue);
}
}
return new LogChangesConfiguration(configuration);
}
public void testGetGroupFromParent() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
String actionConf =
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>";
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(actionConf);
// negative test
final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
workflow.setConf(newConf.toXmlString());
subWorkflow.start(new Context(workflow, action), action);
OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
5000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
subWorkflow.check(new Context(workflow, action), action);
return action.getStatus() == WorkflowActionBean.Status.DONE;
}
});
subWorkflow.check(new Context(workflow, action), action);
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME));
// positive test
newConf.set(OozieClient.GROUP_NAME, getTestGroup());
workflow.setConf(newConf.toXmlString());
final WorkflowActionBean action1 = new WorkflowActionBean();
action1.setConf(actionConf);
action1.setId("W1");
subWorkflow.start(new Context(workflow, action1), action1);
oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL);
waitFor(
5000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
subWorkflow.check(new Context(workflow, action1), action1);
return action1.getStatus() == WorkflowActionBean.Status.DONE;
}
});
subWorkflow.check(new Context(workflow, action1), action1);
subWorkflow.end(new Context(workflow, action1), action1);
wf = oozieClient.getJobInfo(action1.getExternalId());
childConf = new XConfiguration(new StringReader(wf.getConf()));
assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME));
}