private RunningJob submitAction(Context context, Namespace ns) throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
WorkflowAction action = context.getAction();
ae.prepareActionDir(getFileSystem(), context);
ae.submitLauncher(getFileSystem(), context, action);
String jobId = action.getExternalId();
String jobTracker = action.getTrackerUri();
String consoleUrl = action.getConsoleUrl();
assertNotNull(jobId);
assertNotNull(jobTracker);
assertNotNull(consoleUrl);
Element e = XmlUtils.parseXml(action.getConf());
XConfiguration conf =
new XConfiguration(
new StringReader(XmlUtils.prettyPrint(e.getChild("configuration", ns)).toString()));
conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker", ns));
conf.set("fs.default.name", e.getChildTextTrim("name-node", ns));
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf);
final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
assertNotNull(runningJob);
return runningJob;
}
@Override
public void registerForNotification(URI uri, Configuration conf, String user, String actionID)
throws URIHandlerException {
HCatURI hcatURI;
try {
hcatURI = new HCatURI(uri);
} catch (URISyntaxException e) {
throw new URIHandlerException(ErrorCode.E0906, uri, e);
}
HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
if (!hcatService.isRegisteredForNotification(hcatURI)) {
HCatClient client = getHCatClient(uri, conf, user);
try {
String topic = client.getMessageBusTopicName(hcatURI.getDb(), hcatURI.getTable());
if (topic == null) {
return;
}
hcatService.registerForNotification(
hcatURI, topic, new HCatMessageHandler(uri.getAuthority()));
} catch (HCatException e) {
throw new HCatAccessorException(ErrorCode.E1501, e);
} finally {
closeQuietly(client, true);
}
}
PartitionDependencyManagerService pdmService =
Services.get().get(PartitionDependencyManagerService.class);
pdmService.addMissingDependency(hcatURI, actionID);
}
/**
* Get coord job info
*
* @param request servlet request
* @param response servlet response
* @return JsonBean CoordinatorJobBean
* @throws XServletException
* @throws BaseEngineException
*/
private JsonBean getCoordinatorJob(HttpServletRequest request, HttpServletResponse response)
throws XServletException, BaseEngineException {
JsonBean jobBean = null;
CoordinatorEngine coordEngine =
Services.get()
.get(CoordinatorEngineService.class)
.getCoordinatorEngine(getUser(request), getAuthToken(request));
String jobId = getResourceName(request);
String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
String lenStr = request.getParameter(RestConstants.LEN_PARAM);
String filter = request.getParameter(RestConstants.JOB_FILTER_PARAM);
int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
start = (start < 1) ? 1 : start;
// Get default number of coordinator actions to be retrieved
int defaultLen = Services.get().getConf().getInt(COORD_ACTIONS_DEFAULT_LENGTH, 1000);
int len = (lenStr != null) ? Integer.parseInt(lenStr) : 0;
len = (len < 1) ? defaultLen : len;
try {
JsonCoordinatorJob coordJob = coordEngine.getCoordJob(jobId, filter, start, len);
jobBean = coordJob;
} catch (CoordinatorEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
return jobBean;
}
public void testActionCheck() throws Exception {
JPAService jpaService = Services.get().get(JPAService.class);
WorkflowJobBean job =
this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
WorkflowActionBean action =
this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);
WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(action.getId());
new ActionStartXCommand(action.getId(), "map-reduce").call();
action = jpaService.execute(wfActionGetCmd);
ActionExecutorContext context =
new ActionXCommand.ActionExecutorContext(job, action, false, false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
JobConf conf =
actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
String user = conf.get("user.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
String launcherId = action.getExternalId();
final RunningJob launcherJob = jobClient.getJob(JobID.forName(launcherId));
waitFor(
120 * 1000,
new Predicate() {
public boolean evaluate() throws Exception {
return launcherJob.isComplete();
}
});
assertTrue(launcherJob.isSuccessful());
Map<String, String> actionData =
LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
new ActionCheckXCommand(action.getId()).call();
action = jpaService.execute(wfActionGetCmd);
String mapperId = action.getExternalId();
String childId = action.getExternalChildIDs();
assertTrue(launcherId.equals(mapperId));
final RunningJob mrJob = jobClient.getJob(JobID.forName(childId));
waitFor(
120 * 1000,
new Predicate() {
public boolean evaluate() throws Exception {
return mrJob.isComplete();
}
});
assertTrue(mrJob.isSuccessful());
new ActionCheckXCommand(action.getId()).call();
action = jpaService.execute(wfActionGetCmd);
assertEquals("SUCCEEDED", action.getExternalStatus());
}
/**
* Provides functionality to test for set*Data calls not being made by the Action Handler.
*
* @param avoidParam set*Data function call to avoid.
* @param expActionErrorCode the expected action error code.
* @throws Exception
*/
private void _testDataNotSet(String avoidParam, String expActionErrorCode) throws Exception {
String workflowPath = getTestCaseFileUri("workflow.xml");
Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
IOUtils.copyCharStream(reader, writer);
final DagEngine engine = new DagEngine("u");
Configuration conf = new XConfiguration();
conf.set(OozieClient.APP_PATH, workflowPath);
conf.set(OozieClient.USER_NAME, getTestUser());
conf.set(OozieClient.LOG_TOKEN, "t");
conf.set("external-status", "ok");
conf.set("signal-value", "based_on_action_status");
conf.set(avoidParam, "true");
final String jobId = engine.submitJob(conf, true);
final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
store.beginTrx();
Thread.sleep(2000);
waitFor(
5000,
new Predicate() {
public boolean evaluate() throws Exception {
WorkflowJobBean bean = store.getWorkflow(jobId, false);
return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.FAILED);
}
});
store.commitTrx();
store.closeTrx();
final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
store2.beginTrx();
assertEquals(
WorkflowInstance.Status.FAILED,
store2.getWorkflow(jobId, false).getWorkflowInstance().getStatus());
assertEquals(WorkflowJob.Status.FAILED, engine.getJob(jobId).getStatus());
List<WorkflowActionBean> actions = store2.getActionsForWorkflow(jobId, false);
WorkflowActionBean action = null;
for (WorkflowActionBean bean : actions) {
if (bean.getType().equals("test")) {
action = bean;
break;
}
}
assertNotNull(action);
assertEquals(expActionErrorCode, action.getErrorCode());
store2.commitTrx();
store2.closeTrx();
}
/**
* Retrieve registration event
*
* @param jobId the jobId
* @throws CommandException
* @throws JPAExecutorException
*/
public static void updateRegistrationEvent(String jobId)
throws CommandException, JPAExecutorException {
JPAService jpaService = Services.get().get(JPAService.class);
SLAService slaService = Services.get().get(SLAService.class);
try {
SLARegistrationBean reg =
SLARegistrationQueryExecutor.getInstance().get(SLARegQuery.GET_SLA_REG_ALL, jobId);
if (reg != null) { // handle coord rerun with different config without sla
slaService.updateRegistrationEvent(reg);
}
} catch (ServiceException e) {
throw new CommandException(ErrorCode.E1007, " id " + jobId, e.getMessage(), e);
}
}
@Override
protected void setUp() throws Exception {
super.setUp();
services = new Services();
services.init();
jpaService = Services.get().get(JPAService.class);
}
/**
* Start LocalOozie.
*
* @throws Exception if LocalOozie could not be started.
*/
public static synchronized void start() throws Exception {
if (localOozieActive) {
throw new IllegalStateException("LocalOozie is already initialized");
}
String log4jFile = System.getProperty(XLogService.LOG4J_FILE_ENV, null);
String oozieLocalLog = System.getProperty("oozielocal.log", null);
if (log4jFile == null) {
System.setProperty(XLogService.LOG4J_FILE_ENV, "localoozie-log4j.properties");
}
if (oozieLocalLog == null) {
System.setProperty("oozielocal.log", "./oozielocal.log");
}
localOozieActive = true;
new Services().init();
if (log4jFile != null) {
System.setProperty(XLogService.LOG4J_FILE_ENV, log4jFile);
} else {
System.getProperties().remove(XLogService.LOG4J_FILE_ENV);
}
if (oozieLocalLog != null) {
System.setProperty("oozielocal.log", oozieLocalLog);
} else {
System.getProperties().remove("oozielocal.log");
}
container = new EmbeddedServletContainer("oozie");
container.addServletEndpoint("/callback", CallbackServlet.class);
container.start();
String callbackUrl = container.getServletURL("/callback");
Services.get().getConf().set(CallbackService.CONF_BASE_URL, callbackUrl);
XLog.getLog(LocalOozie.class).info("LocalOozie started callback set to [{0}]", callbackUrl);
}
private void checkCoordActions(String jobId, int number, CoordinatorJob.Status status) {
try {
JPAService jpaService = Services.get().get(JPAService.class);
List<CoordinatorActionBean> actions =
jpaService.execute(new CoordJobGetActionsJPAExecutor(jobId));
if (actions.size() != number) {
fail(
"Should have "
+ number
+ " actions created for job "
+ jobId
+ ", but jave "
+ actions.size()
+ " actions.");
}
if (status != null) {
CoordinatorJob job = jpaService.execute(new CoordJobGetJPAExecutor(jobId));
if (job.getStatus() != status) {
fail("Job status " + job.getStatus() + " should be " + status);
}
}
} catch (JPAExecutorException se) {
se.printStackTrace();
fail("Job ID " + jobId + " was not stored properly in db");
}
}
@Test
public void testListenerConfigured() throws Exception {
EventHandlerService ehs = services.get(EventHandlerService.class);
assertNotNull(ehs);
assertTrue(SLAService.isEnabled());
assertTrue(ehs.listEventListeners().contains(SLAJobEventListener.class.getCanonicalName()));
}
/** v1 service implementation to submit a workflow job */
@SuppressWarnings("unchecked")
private JSONObject submitWorkflowJob(HttpServletRequest request, Configuration conf)
throws XServletException {
JSONObject json = new JSONObject();
try {
String action = request.getParameter(RestConstants.ACTION_PARAM);
if (action != null && !action.equals(RestConstants.JOB_ACTION_START)) {
throw new XServletException(
HttpServletResponse.SC_BAD_REQUEST,
ErrorCode.E0303,
RestConstants.ACTION_PARAM,
action);
}
boolean startJob = (action != null);
String user = conf.get(OozieClient.USER_NAME);
DagEngine dagEngine =
Services.get().get(DagEngineService.class).getDagEngine(user, getAuthToken(request));
String id = dagEngine.submitJob(conf, startJob);
json.put(JsonTags.JOB_ID, id);
} catch (DagEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
return json;
}
@SuppressWarnings("unchecked")
private JSONObject getBundleJobs(HttpServletRequest request) throws XServletException {
JSONObject json = new JSONObject();
try {
String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
String lenStr = request.getParameter(RestConstants.LEN_PARAM);
int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
start = (start < 1) ? 1 : start;
int len = (lenStr != null) ? Integer.parseInt(lenStr) : 50;
len = (len < 1) ? 50 : len;
BundleEngine bundleEngine =
Services.get()
.get(BundleEngineService.class)
.getBundleEngine(getUser(request), getAuthToken(request));
BundleJobInfo jobs = bundleEngine.getBundleJobs(filter, start, len);
List<BundleJobBean> jsonJobs = jobs.getBundleJobs();
json.put(JsonTags.BUNDLE_JOBS, BundleJobBean.toJSONArray(jsonJobs));
json.put(JsonTags.BUNDLE_JOB_TOTAL, jobs.getTotal());
json.put(JsonTags.BUNDLE_JOB_OFFSET, jobs.getStart());
json.put(JsonTags.BUNDLE_JOB_LEN, jobs.getLen());
} catch (BundleEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
return json;
}
/**
* Return the default group to which the user belongs.
*
* <p>This implementation always returns 'users'.
*
* @param user user name.
* @return default group of user.
* @throws AuthorizationException thrown if the default group con not be retrieved.
*/
public String getDefaultGroup(String user) throws AuthorizationException {
try {
return Services.get().get(GroupsService.class).getGroups(user).get(0);
} catch (IOException ex) {
throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex);
}
}
/**
* Provides functionality to test errors
*
* @param errorType the error type. (start.non-transient, end.non-transient)
* @param externalStatus the external status to set.
* @param signalValue the signal value to set.
* @throws Exception
*/
private void _testError(String errorType, String externalStatus, String signalValue)
throws Exception {
String workflowPath = getTestCaseFileUri("workflow.xml");
Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
IOUtils.copyCharStream(reader, writer);
final DagEngine engine = new DagEngine("u");
Configuration conf = new XConfiguration();
conf.set(OozieClient.APP_PATH, workflowPath);
conf.set(OozieClient.USER_NAME, getTestUser());
conf.set(OozieClient.LOG_TOKEN, "t");
conf.set("error", errorType);
conf.set("external-status", externalStatus);
conf.set("signal-value", signalValue);
final String jobId = engine.submitJob(conf, true);
final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
store.beginTrx();
waitFor(
5000,
new Predicate() {
public boolean evaluate() throws Exception {
WorkflowJobBean bean = store.getWorkflow(jobId, false);
return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
}
});
assertEquals(WorkflowJob.Status.KILLED, engine.getJob(jobId).getStatus());
store.commitTrx();
store.closeTrx();
}
/**
* Rerun bundle job
*
* @param request servlet request
* @param response servlet response
* @param conf configration object
* @throws XServletException
*/
private void rerunBundleJob(
HttpServletRequest request, HttpServletResponse response, Configuration conf)
throws XServletException {
JSONObject json = new JSONObject();
BundleEngine bundleEngine =
Services.get()
.get(BundleEngineService.class)
.getBundleEngine(getUser(request), getAuthToken(request));
String jobId = getResourceName(request);
String coordScope = request.getParameter(RestConstants.JOB_BUNDLE_RERUN_COORD_SCOPE_PARAM);
String dateScope = request.getParameter(RestConstants.JOB_BUNDLE_RERUN_DATE_SCOPE_PARAM);
String refresh = request.getParameter(RestConstants.JOB_COORD_RERUN_REFRESH_PARAM);
String noCleanup = request.getParameter(RestConstants.JOB_COORD_RERUN_NOCLEANUP_PARAM);
XLog.getLog(getClass())
.info(
"Rerun Bundle for jobId="
+ jobId
+ ", coordScope="
+ coordScope
+ ", dateScope="
+ dateScope
+ ", refresh="
+ refresh
+ ", noCleanup="
+ noCleanup);
try {
bundleEngine.reRun(
jobId, coordScope, dateScope, Boolean.valueOf(refresh), Boolean.valueOf(noCleanup));
} catch (BaseEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
}
/**
* Load the list of admin users from {@link AuthorizationService#ADMIN_USERS_FILE}
*
* @throws ServiceException if the admin user list could not be loaded.
*/
private void loadAdminUsers() throws ServiceException {
String configDir = Services.get().get(ConfigurationService.class).getConfigDir();
if (configDir != null) {
File file = new File(configDir, ADMIN_USERS_FILE);
if (file.exists()) {
try {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
try {
String line = br.readLine();
while (line != null) {
line = line.trim();
if (line.length() > 0 && !line.startsWith("#")) {
adminUsers.add(line);
}
line = br.readLine();
}
} catch (IOException ex) {
throw new ServiceException(ErrorCode.E0160, file.getAbsolutePath(), ex);
}
} catch (FileNotFoundException ex) {
throw new ServiceException(ErrorCode.E0160, ex);
}
} else {
log.warn(
"Admin users file not available in config dir [{0}], running without admin users",
configDir);
}
} else {
log.warn("Reading configuration from classpath, running without admin users");
}
}
/**
* Basic test
*
* @throws Exception
*/
public void testBasicSubmit() throws Exception {
Configuration conf = new XConfiguration();
String appPath = getTestCaseDir();
String appXml =
"<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" "
+ "xmlns=\"uri:oozie:coordinator:0.1\"> <controls> <concurrency>2</concurrency> "
+ "<execution>LIFO</execution> </controls> <datasets> "
+ "<dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\" "
+ "timezone=\"UTC\"> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> "
+ "<dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\" "
+ "timezone=\"UTC\"> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> "
+ "</datasets> <input-events> "
+ "<data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in> "
+ "</input-events> "
+ "<output-events> <data-out name=\"LOCAL_A\" dataset=\"local_a\"> "
+ "<instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> "
+ "<configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> "
+ "<property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> "
+ "</property></configuration> </workflow> </action> </coordinator-app>";
writeToFile(appXml, appPath);
conf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
conf.set(OozieClient.USER_NAME, getTestUser());
conf.set(OozieClient.GROUP_NAME, "other");
CoordSubmitCommand sc = new CoordSubmitCommand(conf, "UNIT_TESTING");
String jobId = sc.call();
assertEquals(jobId.substring(jobId.length() - 2), "-C");
CoordinatorJobBean job = checkCoordJobs(jobId);
if (job != null) {
assertEquals(
job.getTimeout(),
Services.get().getConf().getInt("oozie.service.coord.normal.default.timeout", -2));
}
}
protected CoordinatorJobBean addRecordToCoordJobTable(
CoordinatorJob.Status status, Date startTime, Date endTime, Date pauseTime, int timeout)
throws Exception {
CoordinatorJobBean coordJob = createCoordJob(status);
coordJob.setStartTime(startTime);
coordJob.setEndTime(endTime);
coordJob.setPauseTime(pauseTime);
coordJob.setFrequency(5);
coordJob.setTimeUnit(Timeunit.MINUTE);
coordJob.setTimeout(timeout);
coordJob.setConcurrency(3);
try {
JPAService jpaService = Services.get().get(JPAService.class);
assertNotNull(jpaService);
CoordJobInsertJPAExecutor coordInsertCmd = new CoordJobInsertJPAExecutor(coordJob);
jpaService.execute(coordInsertCmd);
} catch (JPAExecutorException ex) {
ex.printStackTrace();
fail("Unable to insert the test coord job record to table");
throw ex;
}
return coordJob;
}
protected FileSystem getAppFileSystem(WorkflowJob workflow)
throws HadoopAccessorException, IOException, URISyntaxException {
URI uri = new URI(workflow.getAppPath());
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
Configuration fsConf = has.createJobConf(uri.getAuthority());
return has.createFileSystem(workflow.getUser(), uri, fsConf);
}
/**
* Test : verify the PreconditionException is thrown when actionCheckDelay > 0
*
* @throws Exception
*/
public void testActionCheckPreCondition1() throws Exception {
Instrumentation inst = Services.get().get(InstrumentationService.class).get();
WorkflowJobBean job =
this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
WorkflowActionBean action =
this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);
ActionCheckXCommand checkCmd = new ActionCheckXCommand(action.getId(), 10);
long counterVal;
try {
counterVal =
inst.getCounters()
.get(XCommand.INSTRUMENTATION_GROUP)
.get(checkCmd.getName() + ".preconditionfailed")
.getValue();
} catch (NullPointerException e) {
// counter might be null
counterVal = 0L;
}
assertEquals(0L, counterVal);
checkCmd.call();
// precondition failed because of actionCheckDelay > 0
counterVal =
inst.getCounters()
.get(XCommand.INSTRUMENTATION_GROUP)
.get(checkCmd.getName() + ".preconditionfailed")
.getValue();
assertEquals(1L, counterVal);
}
public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
for (Map.Entry<String, String> entry : conf) {
eval.setVariable(entry.getKey(), entry.getValue());
}
return eval;
}
/**
* v1 service implementation to get a list of workflows, with filtering or interested windows
* embedded in the request object
*/
private JSONObject getWorkflowJobs(HttpServletRequest request) throws XServletException {
JSONObject json = new JSONObject();
try {
String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
String lenStr = request.getParameter(RestConstants.LEN_PARAM);
int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
start = (start < 1) ? 1 : start;
int len = (lenStr != null) ? Integer.parseInt(lenStr) : 50;
len = (len < 1) ? 50 : len;
DagEngine dagEngine =
Services.get()
.get(DagEngineService.class)
.getDagEngine(getUser(request), getAuthToken(request));
WorkflowsInfo jobs = dagEngine.getJobs(filter, start, len);
List<WorkflowJobBean> jsonWorkflows = jobs.getWorkflows();
json.put(JsonTags.WORKFLOWS_JOBS, WorkflowJobBean.toJSONArray(jsonWorkflows));
json.put(JsonTags.WORKFLOWS_TOTAL, jobs.getTotal());
json.put(JsonTags.WORKFLOWS_OFFSET, jobs.getStart());
json.put(JsonTags.WORKFLOWS_LEN, jobs.getLen());
} catch (DagEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
return json;
}
/**
* Check if the user belongs to the group or not.
*
* @param user user name.
* @param group group name.
* @return if the user belongs to the group or not.
* @throws AuthorizationException thrown if the authorization query can not be performed.
*/
protected boolean isUserInGroup(String user, String group) throws AuthorizationException {
GroupsService groupsService = Services.get().get(GroupsService.class);
try {
return groupsService.getGroups(user).contains(group);
} catch (IOException ex) {
throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex);
}
}
/**
* Return a {@link org.apache.oozie.client.OozieClient} for LocalOozie configured for a given
* user.
*
* <p>The following methods of the client are NOP in the returned instance: {@link
* org.apache.oozie.client.OozieClient#validateWSVersion}, {@link
* org.apache.oozie.client.OozieClient#setHeader}, {@link
* org.apache.oozie.client.OozieClient#getHeader}, {@link
* org.apache.oozie.client.OozieClient#removeHeader}, {@link
* org.apache.oozie.client.OozieClient#getHeaderNames} and {@link
* org.apache.oozie.client.OozieClient#setSafeMode}.
*
* @param user user name to use in LocalOozie for running workflows.
* @return a {@link org.apache.oozie.client.OozieClient} for LocalOozie configured for the given
* user.
*/
public static OozieClient getClient(String user) {
if (!localOozieActive) {
throw new IllegalStateException("LocalOozie is not initialized");
}
ParamChecker.notEmpty(user, "user");
DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, "undef");
return new LocalOozieClient(dagEngine);
}
private ConnectionContext getConnectionContext() {
Configuration conf = services.getConf();
String jmsProps = conf.get(JMSJobEventListener.JMS_CONNECTION_PROPERTIES);
JMSConnectionInfo connInfo = new JMSConnectionInfo(jmsProps);
JMSAccessorService jmsService = Services.get().get(JMSAccessorService.class);
ConnectionContext jmsContext = jmsService.createConnectionContext(connInfo);
return jmsContext;
}
private CoordinatorActionBean createCoordinatorActionBean(CoordinatorJob job) throws IOException {
CoordinatorActionBean actionBean = new CoordinatorActionBean();
String actionId = Services.get().get(UUIDService.class).generateChildId(job.getId(), "1");
actionBean.setJobId(job.getId());
actionBean.setId(actionId);
Configuration jobConf = new XConfiguration(new StringReader(job.getConf()));
actionBean.setRunConf(XmlUtils.prettyPrint(jobConf).toString());
return actionBean;
}
public void testSubworkflowLib() throws Exception {
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
FileSystem fs = getFileSystem();
Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
fs.create(parentLibJar);
assertTrue(fs.exists(parentLibJar));
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
workflow.setConf(newConf.toXmlString());
Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
fs.create(subwfLibJar);
assertTrue(fs.exists(subwfLibJar));
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
childConf = wps.createProtoActionConf(childConf, "authToken", true);
assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
}
protected WorkflowActionBean createWorkflowActionSetPending(
String wfId, WorkflowAction.Status status) throws Exception {
WorkflowActionBean action = new WorkflowActionBean();
String actionname = "testAction";
action.setName(actionname);
action.setCred("null");
action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname));
action.setJobId(wfId);
action.setType("map-reduce");
action.setTransition("transition");
action.setStatus(status);
action.setStartTime(new Date());
action.setEndTime(new Date());
action.setLastCheckTime(new Date());
action.setPending();
action.setExecutionPath("/");
Path inputDir = new Path(getFsTestCaseDir(), "input");
Path outputDir = new Path(getFsTestCaseDir(), "output");
FileSystem fs = getFileSystem();
Writer w = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
w.write("dummy\n");
w.write("dummy\n");
w.close();
String actionXml =
"<map-reduce>"
+ "<job-tracker>"
+ getJobTrackerUri()
+ "</job-tracker>"
+ "<name-node>"
+ getNameNodeUri()
+ "</name-node>"
+ "<prepare><delete path=\""
+ outputDir.toString()
+ "\"/></prepare>"
+ "<configuration>"
+ "<property><name>mapred.mapper.class</name><value>"
+ MapperReducerForTest.class.getName()
+ "</value></property>"
+ "<property><name>mapred.reducer.class</name><value>"
+ MapperReducerForTest.class.getName()
+ "</value></property>"
+ "<property><name>mapred.input.dir</name><value>"
+ inputDir.toString()
+ "</value></property>"
+ "<property><name>mapred.output.dir</name><value>"
+ outputDir.toString()
+ "</value></property>"
+ "</configuration>"
+ "</map-reduce>";
action.setConf(actionXml);
return action;
}
private void checkCoordActionsTimeout(String actionId, int expected) {
try {
JPAService jpaService = Services.get().get(JPAService.class);
CoordinatorActionBean action = jpaService.execute(new CoordActionGetJPAExecutor(actionId));
assertEquals(action.getTimeOut(), expected);
} catch (JPAExecutorException se) {
se.printStackTrace();
fail("Action ID " + actionId + " was not stored properly in db");
}
}
/**
* Check whether a URI path exists
*
* @param sPath
* @param conf
* @return
* @throws IOException
*/
private static boolean isPathAvailable(
String sPath, String user, String group, Configuration conf)
throws IOException, HadoopAccessorException {
// sPath += "/" + END_OF_OPERATION_INDICATOR_FILE;
Path path = new Path(sPath);
return Services.get()
.get(HadoopAccessorService.class)
.createFileSystem(user, group, path.toUri(), conf)
.exists(path);
}