/**
* Initialize the service.
*
* @param services services instance.
* @throws ServiceException thrown if the service could not be initialized.
*/
public void init(Services services) throws ServiceException {
try {
wfSchema = loadSchema(services.getConf(), OOZIE_WORKFLOW_XSD, WF_CONF_EXT_SCHEMAS);
coordSchema = loadSchema(services.getConf(), OOZIE_COORDINATOR_XSD, COORD_CONF_EXT_SCHEMAS);
bundleSchema = loadSchema(services.getConf(), OOZIE_BUNDLE_XSD, BUNDLE_CONF_EXT_SCHEMAS);
slaSchema = loadSchema(services.getConf(), OOZIE_SLA_SEMANTIC_XSD, SLA_CONF_EXT_SCHEMAS);
bundleSchema = loadSchema(services.getConf(), OOZIE_BUNDLE_XSD, BUNDLE_CONF_EXT_SCHEMAS);
} catch (SAXException ex) {
throw new ServiceException(ErrorCode.E0130, ex.getMessage(), ex);
} catch (IOException ex) {
throw new ServiceException(ErrorCode.E0131, ex.getMessage(), ex);
}
}
@Override
public void init(Services services) throws ServiceException {
try {
init(services.getConf());
} catch (Exception e) {
throw new ServiceException(ErrorCode.E0902, e);
}
}
private ConnectionContext getConnectionContext() {
Configuration conf = services.getConf();
String jmsProps = conf.get(JMSJobEventListener.JMS_CONNECTION_PROPERTIES);
JMSConnectionInfo connInfo = new JMSConnectionInfo(jmsProps);
JMSAccessorService jmsService = Services.get().get(JMSAccessorService.class);
ConnectionContext jmsContext = jmsService.createConnectionContext(connInfo);
return jmsContext;
}
/**
* Initialize the EL service.
*
* @param services services instance.
* @throws ServiceException thrown if the EL service could not be initialized.
*/
@Override
public synchronized void init(Services services) throws ServiceException {
log.trace("Constants and functions registration");
constants = new HashMap<String, List<ELConstant>>();
functions = new HashMap<String, List<ELFunction>>();
// Get the list of group names from configuration file
// defined in the property tag: oozie.service.ELSerice.groups
// String []groupList = services.getConf().get(CONF_GROUPS, "").trim().split(",");
String[] groupList = ConfigurationService.getStrings(services.getConf(), CONF_GROUPS);
// For each group, collect the required functions and constants
// and store it into HashMap
for (String group : groupList) {
List<ELConstant> tmpConstants = new ArrayList<ELConstant>();
tmpConstants.addAll(extractConstants(services.getConf(), CONF_CONSTANTS + group));
tmpConstants.addAll(extractConstants(services.getConf(), CONF_EXT_CONSTANTS + group));
constants.put(group, tmpConstants);
List<ELFunction> tmpFunctions = new ArrayList<ELFunction>();
tmpFunctions.addAll(extractFunctions(services.getConf(), CONF_FUNCTIONS + group));
tmpFunctions.addAll(extractFunctions(services.getConf(), CONF_EXT_FUNCTIONS + group));
functions.put(group, tmpFunctions);
}
}
@Override
@Before
protected void setUp() throws Exception {
super.setUp();
services = new Services();
Configuration conf = services.getConf();
conf.set(
Services.CONF_SERVICE_EXT_CLASSES,
"org.apache.oozie.service.EventHandlerService,"
+ "org.apache.oozie.sla.service.SLAService");
conf.setClass(
EventHandlerService.CONF_LISTENERS, SLAJobEventListener.class, JobEventListener.class);
services.init();
}
@Before
@Override
protected void setUp() throws Exception {
super.setUp();
services = new Services();
conf = services.getConf();
conf.set(
Services.CONF_SERVICE_EXT_CLASSES,
JMSAccessorService.class.getName() + "," + JMSTopicService.class.getName());
conf.set(
JMSJobEventListener.JMS_CONNECTION_PROPERTIES,
"java.naming.factory.initial#"
+ ActiveMQConnFactory
+ ";"
+ "java.naming.provider.url#"
+ localActiveMQBroker
+ ";"
+ "connectionFactoryNames#"
+ "ConnectionFactory");
services.init();
}
/**
* Initialize the service.
*
* <p>Reads the security related configuration. parameters - security enabled and list of super
* users.
*
* @param services services instance.
* @throws ServiceException thrown if the service could not be initialized.
*/
public void init(Services services) throws ServiceException {
authorizationEnabled =
ConfigUtils.getWithDeprecatedCheck(
services.getConf(), CONF_AUTHORIZATION_ENABLED, CONF_SECURITY_ENABLED, false);
if (authorizationEnabled) {
log.info("Oozie running with authorization enabled");
useDefaultGroupAsAcl = Services.get().getConf().getBoolean(CONF_DEFAULT_GROUP_AS_ACL, false);
String[] str = getTrimmedStrings(Services.get().getConf().get(CONF_ADMIN_GROUPS));
if (str.length > 0) {
log.info("Admin users will be checked against the defined admin groups");
adminGroups = new HashSet<String>();
for (String s : str) {
adminGroups.add(s.trim());
}
} else {
log.info("Admin users will be checked against the 'adminusers.txt' file contents");
adminUsers = new HashSet<String>();
loadAdminUsers();
}
} else {
log.warn("Oozie running with authorization disabled");
}
instrumentation = Services.get().get(InstrumentationService.class).get();
}
public void testActionCheckTransientDuringMRAction() throws Exception {
// When using YARN, skip this test because it relies on shutting down the job tracker, which
// isn't used in YARN
if (createJobConf().get("yarn.resourcemanager.address") != null) {
return;
}
services.destroy();
// Make the max number of retries lower so the test won't take as long
final int maxRetries = 2;
setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
services = new Services();
// Disable ActionCheckerService so it doesn't interfere by triggering any extra
// ActionCheckXCommands
setClassesToBeExcluded(
services.getConf(), new String[] {"org.apache.oozie.service.ActionCheckerService"});
services.init();
final JPAService jpaService = Services.get().get(JPAService.class);
WorkflowJobBean job0 =
this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
final String jobId = job0.getId();
WorkflowActionBean action0 =
this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
final String actionId = action0.getId();
final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);
new ActionStartXCommand(actionId, "map-reduce").call();
final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
String originalLauncherId = action1.getExternalId();
ActionExecutorContext context =
new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
JobConf conf =
actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
String user = conf.get("user.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId));
waitFor(
120 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return launcherJob.isComplete();
}
});
assertTrue(launcherJob.isSuccessful());
Map<String, String> actionData =
LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
new ActionCheckXCommand(action1.getId()).call();
WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
String originalMapperId = action2.getExternalChildIDs();
assertFalse(originalLauncherId.equals(originalMapperId));
// At this point, the launcher job has finished and the map-reduce action has started (but not
// finished)
// Now, shutdown the job tracker to pretend it has gone down during the map-reduce job
executeWhileJobTrackerIsShutdown(
new ShutdownJobTrackerExecutable() {
@Override
public void execute() throws Exception {
assertEquals(0, action1.getRetries());
new ActionCheckXCommand(actionId).call();
waitFor(
30 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
return (action1a.getRetries() > 0);
}
});
waitFor(
180 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
return (action1a.getRetries() == 0);
}
});
WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
assertEquals(0, action1b.getRetries());
assertEquals("START_MANUAL", action1b.getStatusStr());
WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
assertEquals("SUSPENDED", job1.getStatusStr());
// At this point, the action has gotten a transient error, even after maxRetries tries
// so the workflow has been
// SUSPENDED
}
});
// Now, lets bring the job tracker back up and resume the workflow (which will restart the
// current action)
// It should now continue and finish with SUCCEEDED
new ResumeXCommand(jobId).call();
WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
assertEquals("RUNNING", job2.getStatusStr());
sleep(500);
new ActionCheckXCommand(actionId).call();
WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
String launcherId = action3.getExternalId();
assertFalse(originalLauncherId.equals(launcherId));
final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId));
waitFor(
120 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return launcherJob2.isComplete();
}
});
assertTrue(launcherJob2.isSuccessful());
actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
new ActionCheckXCommand(actionId).call();
WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
String mapperId = action4.getExternalChildIDs();
assertFalse(originalMapperId.equals(mapperId));
final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));
waitFor(
120 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return mrJob.isComplete();
}
});
assertTrue(mrJob.isSuccessful());
new ActionCheckXCommand(actionId).call();
WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);
assertEquals("SUCCEEDED", action5.getExternalStatus());
}
@Test
public void testOnJobEvent() throws Exception {
SLAService slas = services.get(SLAService.class);
SLAJobEventListener listener = new SLAJobEventListener();
listener.init(services.getConf());
// add dummy registration events to the SLAService map
SLARegistrationBean job = _createSLARegBean("wf1", AppType.WORKFLOW_JOB);
job.setExpectedStart(DateUtils.parseDateUTC("2012-07-22T00:00Z"));
slas.addRegistrationEvent(job);
assertEquals(1, slas.getSLACalculator().size());
Date actualStart = DateUtils.parseDateUTC("2012-07-22T01:00Z");
WorkflowJobEvent wfe =
new WorkflowJobEvent(
"wf1", "caId1", WorkflowJob.Status.RUNNING, "user1", "wf-app-name1", actualStart, null);
listener.onWorkflowJobEvent(wfe);
SLACalcStatus serviceObj = slas.getSLACalculator().get("wf1");
// check that start sla has been calculated
assertEquals(EventStatus.START_MISS, serviceObj.getEventStatus());
assertEquals(1, serviceObj.getEventProcessed()); // Job switching to running is only partially
// sla processed. so state = 1
job = _createSLARegBean("wfId1@wa1", AppType.WORKFLOW_ACTION);
slas.addRegistrationEvent(job);
assertEquals(2, slas.getSLACalculator().size());
job.setExpectedStart(DateUtils.parseDateUTC("2012-07-22T00:00Z"));
WorkflowActionEvent wae =
new WorkflowActionEvent(
"wfId1@wa1",
"wfId1",
WorkflowAction.Status.RUNNING,
"user1",
"wf-app-name1",
actualStart,
null);
listener.onWorkflowActionEvent(wae);
serviceObj = slas.getSLACalculator().get("wfId1@wa1");
// check that start sla has been calculated
assertEquals(EventStatus.START_MISS, serviceObj.getEventStatus());
job = _createSLARegBean("cj1", AppType.COORDINATOR_JOB);
job.setExpectedEnd(DateUtils.parseDateUTC("2012-07-22T01:00Z"));
slas.addRegistrationEvent(job);
assertEquals(3, slas.getSLACalculator().size());
Date actualEnd = DateUtils.parseDateUTC("2012-07-22T00:00Z");
CoordinatorJobEvent cje =
new CoordinatorJobEvent(
"cj1",
"bj1",
CoordinatorJob.Status.SUCCEEDED,
"user1",
"coord-app-name1",
actualStart,
actualEnd);
listener.onCoordinatorJobEvent(cje);
SLASummaryBean summary =
SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, "cj1");
// check that end and duration sla has been calculated
assertEquals(6, summary.getEventProcessed());
assertEquals(EventStatus.END_MET, summary.getEventStatus());
job = _createSLARegBean("cj1@ca1", AppType.COORDINATOR_ACTION);
actualEnd = DateUtils.parseDateUTC("2012-07-22T02:00Z");
slas.addRegistrationEvent(job);
assertEquals(4, slas.getSLACalculator().size());
CoordinatorActionEvent cae =
new CoordinatorActionEvent(
"cj1@ca1",
"cj1",
CoordinatorAction.Status.RUNNING,
"user1",
"coord-app-name1",
null,
actualEnd,
null);
listener.onCoordinatorActionEvent(cae);
cae =
new CoordinatorActionEvent(
"cj1@ca1",
"cj1",
CoordinatorAction.Status.KILLED,
"user1",
"coord-app-name1",
null,
actualEnd,
null);
listener.onCoordinatorActionEvent(cae);
summary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, "cj1@ca1");
// check that all events are processed
assertEquals(8, summary.getEventProcessed());
assertEquals(EventStatus.END_MISS, summary.getEventStatus());
assertEquals(3, slas.getSLACalculator().size());
}