Esempio n. 1
0
 /**
  * Initialize the service.
  *
  * @param services services instance.
  * @throws ServiceException thrown if the service could not be initialized.
  */
 public void init(Services services) throws ServiceException {
   try {
     wfSchema = loadSchema(services.getConf(), OOZIE_WORKFLOW_XSD, WF_CONF_EXT_SCHEMAS);
     coordSchema = loadSchema(services.getConf(), OOZIE_COORDINATOR_XSD, COORD_CONF_EXT_SCHEMAS);
     bundleSchema = loadSchema(services.getConf(), OOZIE_BUNDLE_XSD, BUNDLE_CONF_EXT_SCHEMAS);
     slaSchema = loadSchema(services.getConf(), OOZIE_SLA_SEMANTIC_XSD, SLA_CONF_EXT_SCHEMAS);
     bundleSchema = loadSchema(services.getConf(), OOZIE_BUNDLE_XSD, BUNDLE_CONF_EXT_SCHEMAS);
   } catch (SAXException ex) {
     throw new ServiceException(ErrorCode.E0130, ex.getMessage(), ex);
   } catch (IOException ex) {
     throw new ServiceException(ErrorCode.E0131, ex.getMessage(), ex);
   }
 }
Esempio n. 2
0
 @Override
 public void init(Services services) throws ServiceException {
   try {
     init(services.getConf());
   } catch (Exception e) {
     throw new ServiceException(ErrorCode.E0902, e);
   }
 }
 private ConnectionContext getConnectionContext() {
   Configuration conf = services.getConf();
   String jmsProps = conf.get(JMSJobEventListener.JMS_CONNECTION_PROPERTIES);
   JMSConnectionInfo connInfo = new JMSConnectionInfo(jmsProps);
   JMSAccessorService jmsService = Services.get().get(JMSAccessorService.class);
   ConnectionContext jmsContext = jmsService.createConnectionContext(connInfo);
   return jmsContext;
 }
Esempio n. 4
0
 /**
  * Initialize the EL service.
  *
  * @param services services instance.
  * @throws ServiceException thrown if the EL service could not be initialized.
  */
 @Override
 public synchronized void init(Services services) throws ServiceException {
   log.trace("Constants and functions registration");
   constants = new HashMap<String, List<ELConstant>>();
   functions = new HashMap<String, List<ELFunction>>();
   // Get the list of group names from configuration file
   // defined in the property tag: oozie.service.ELSerice.groups
   // String []groupList = services.getConf().get(CONF_GROUPS, "").trim().split(",");
   String[] groupList = ConfigurationService.getStrings(services.getConf(), CONF_GROUPS);
   // For each group, collect the required functions and constants
   // and store it into HashMap
   for (String group : groupList) {
     List<ELConstant> tmpConstants = new ArrayList<ELConstant>();
     tmpConstants.addAll(extractConstants(services.getConf(), CONF_CONSTANTS + group));
     tmpConstants.addAll(extractConstants(services.getConf(), CONF_EXT_CONSTANTS + group));
     constants.put(group, tmpConstants);
     List<ELFunction> tmpFunctions = new ArrayList<ELFunction>();
     tmpFunctions.addAll(extractFunctions(services.getConf(), CONF_FUNCTIONS + group));
     tmpFunctions.addAll(extractFunctions(services.getConf(), CONF_EXT_FUNCTIONS + group));
     functions.put(group, tmpFunctions);
   }
 }
 @Override
 @Before
 protected void setUp() throws Exception {
   super.setUp();
   services = new Services();
   Configuration conf = services.getConf();
   conf.set(
       Services.CONF_SERVICE_EXT_CLASSES,
       "org.apache.oozie.service.EventHandlerService,"
           + "org.apache.oozie.sla.service.SLAService");
   conf.setClass(
       EventHandlerService.CONF_LISTENERS, SLAJobEventListener.class, JobEventListener.class);
   services.init();
 }
 @Before
 @Override
 protected void setUp() throws Exception {
   super.setUp();
   services = new Services();
   conf = services.getConf();
   conf.set(
       Services.CONF_SERVICE_EXT_CLASSES,
       JMSAccessorService.class.getName() + "," + JMSTopicService.class.getName());
   conf.set(
       JMSJobEventListener.JMS_CONNECTION_PROPERTIES,
       "java.naming.factory.initial#"
           + ActiveMQConnFactory
           + ";"
           + "java.naming.provider.url#"
           + localActiveMQBroker
           + ";"
           + "connectionFactoryNames#"
           + "ConnectionFactory");
   services.init();
 }
Esempio n. 7
0
 /**
  * Initialize the service.
  *
  * <p>Reads the security related configuration. parameters - security enabled and list of super
  * users.
  *
  * @param services services instance.
  * @throws ServiceException thrown if the service could not be initialized.
  */
 public void init(Services services) throws ServiceException {
   authorizationEnabled =
       ConfigUtils.getWithDeprecatedCheck(
           services.getConf(), CONF_AUTHORIZATION_ENABLED, CONF_SECURITY_ENABLED, false);
   if (authorizationEnabled) {
     log.info("Oozie running with authorization enabled");
     useDefaultGroupAsAcl = Services.get().getConf().getBoolean(CONF_DEFAULT_GROUP_AS_ACL, false);
     String[] str = getTrimmedStrings(Services.get().getConf().get(CONF_ADMIN_GROUPS));
     if (str.length > 0) {
       log.info("Admin users will be checked against the defined admin groups");
       adminGroups = new HashSet<String>();
       for (String s : str) {
         adminGroups.add(s.trim());
       }
     } else {
       log.info("Admin users will be checked against the 'adminusers.txt' file contents");
       adminUsers = new HashSet<String>();
       loadAdminUsers();
     }
   } else {
     log.warn("Oozie running with authorization disabled");
   }
   instrumentation = Services.get().get(InstrumentationService.class).get();
 }
  public void testActionCheckTransientDuringMRAction() throws Exception {
    // When using YARN, skip this test because it relies on shutting down the job tracker, which
    // isn't used in YARN
    if (createJobConf().get("yarn.resourcemanager.address") != null) {
      return;
    }
    services.destroy();
    // Make the max number of retries lower so the test won't take as long
    final int maxRetries = 2;
    setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
    services = new Services();
    // Disable ActionCheckerService so it doesn't interfere by triggering any extra
    // ActionCheckXCommands
    setClassesToBeExcluded(
        services.getConf(), new String[] {"org.apache.oozie.service.ActionCheckerService"});
    services.init();

    final JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job0 =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    final String jobId = job0.getId();
    WorkflowActionBean action0 =
        this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
    final String actionId = action0.getId();
    final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);

    new ActionStartXCommand(actionId, "map-reduce").call();
    final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
    String originalLauncherId = action1.getExternalId();

    ActionExecutorContext context =
        new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    JobConf conf =
        actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient =
        Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);

    final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
          }
        });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData =
        LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(action1.getId()).call();
    WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
    String originalMapperId = action2.getExternalChildIDs();

    assertFalse(originalLauncherId.equals(originalMapperId));

    // At this point, the launcher job has finished and the map-reduce action has started (but not
    // finished)
    // Now, shutdown the job tracker to pretend it has gone down during the map-reduce job
    executeWhileJobTrackerIsShutdown(
        new ShutdownJobTrackerExecutable() {
          @Override
          public void execute() throws Exception {
            assertEquals(0, action1.getRetries());
            new ActionCheckXCommand(actionId).call();

            waitFor(
                30 * 1000,
                new Predicate() {
                  @Override
                  public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() > 0);
                  }
                });
            waitFor(
                180 * 1000,
                new Predicate() {
                  @Override
                  public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() == 0);
                  }
                });
            WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
            assertEquals(0, action1b.getRetries());
            assertEquals("START_MANUAL", action1b.getStatusStr());

            WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
            assertEquals("SUSPENDED", job1.getStatusStr());

            // At this point, the action has gotten a transient error, even after maxRetries tries
            // so the workflow has been
            // SUSPENDED
          }
        });
    // Now, lets bring the job tracker back up and resume the workflow (which will restart the
    // current action)
    // It should now continue and finish with SUCCEEDED
    new ResumeXCommand(jobId).call();
    WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    assertEquals("RUNNING", job2.getStatusStr());

    sleep(500);

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
    String launcherId = action3.getExternalId();

    assertFalse(originalLauncherId.equals(launcherId));

    final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return launcherJob2.isComplete();
          }
        });

    assertTrue(launcherJob2.isSuccessful());
    actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
    String mapperId = action4.getExternalChildIDs();
    assertFalse(originalMapperId.equals(mapperId));

    final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return mrJob.isComplete();
          }
        });
    assertTrue(mrJob.isSuccessful());

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);

    assertEquals("SUCCEEDED", action5.getExternalStatus());
  }
  @Test
  public void testOnJobEvent() throws Exception {
    SLAService slas = services.get(SLAService.class);
    SLAJobEventListener listener = new SLAJobEventListener();
    listener.init(services.getConf());
    // add dummy registration events to the SLAService map
    SLARegistrationBean job = _createSLARegBean("wf1", AppType.WORKFLOW_JOB);
    job.setExpectedStart(DateUtils.parseDateUTC("2012-07-22T00:00Z"));
    slas.addRegistrationEvent(job);
    assertEquals(1, slas.getSLACalculator().size());
    Date actualStart = DateUtils.parseDateUTC("2012-07-22T01:00Z");
    WorkflowJobEvent wfe =
        new WorkflowJobEvent(
            "wf1", "caId1", WorkflowJob.Status.RUNNING, "user1", "wf-app-name1", actualStart, null);
    listener.onWorkflowJobEvent(wfe);
    SLACalcStatus serviceObj = slas.getSLACalculator().get("wf1");
    // check that start sla has been calculated
    assertEquals(EventStatus.START_MISS, serviceObj.getEventStatus());
    assertEquals(1, serviceObj.getEventProcessed()); // Job switching to running is only partially
    // sla processed. so state = 1

    job = _createSLARegBean("wfId1@wa1", AppType.WORKFLOW_ACTION);
    slas.addRegistrationEvent(job);
    assertEquals(2, slas.getSLACalculator().size());
    job.setExpectedStart(DateUtils.parseDateUTC("2012-07-22T00:00Z"));
    WorkflowActionEvent wae =
        new WorkflowActionEvent(
            "wfId1@wa1",
            "wfId1",
            WorkflowAction.Status.RUNNING,
            "user1",
            "wf-app-name1",
            actualStart,
            null);
    listener.onWorkflowActionEvent(wae);
    serviceObj = slas.getSLACalculator().get("wfId1@wa1");
    // check that start sla has been calculated
    assertEquals(EventStatus.START_MISS, serviceObj.getEventStatus());

    job = _createSLARegBean("cj1", AppType.COORDINATOR_JOB);
    job.setExpectedEnd(DateUtils.parseDateUTC("2012-07-22T01:00Z"));
    slas.addRegistrationEvent(job);
    assertEquals(3, slas.getSLACalculator().size());
    Date actualEnd = DateUtils.parseDateUTC("2012-07-22T00:00Z");
    CoordinatorJobEvent cje =
        new CoordinatorJobEvent(
            "cj1",
            "bj1",
            CoordinatorJob.Status.SUCCEEDED,
            "user1",
            "coord-app-name1",
            actualStart,
            actualEnd);
    listener.onCoordinatorJobEvent(cje);

    SLASummaryBean summary =
        SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, "cj1");
    // check that end and duration sla has been calculated
    assertEquals(6, summary.getEventProcessed());

    assertEquals(EventStatus.END_MET, summary.getEventStatus());

    job = _createSLARegBean("cj1@ca1", AppType.COORDINATOR_ACTION);
    actualEnd = DateUtils.parseDateUTC("2012-07-22T02:00Z");
    slas.addRegistrationEvent(job);
    assertEquals(4, slas.getSLACalculator().size());
    CoordinatorActionEvent cae =
        new CoordinatorActionEvent(
            "cj1@ca1",
            "cj1",
            CoordinatorAction.Status.RUNNING,
            "user1",
            "coord-app-name1",
            null,
            actualEnd,
            null);
    listener.onCoordinatorActionEvent(cae);
    cae =
        new CoordinatorActionEvent(
            "cj1@ca1",
            "cj1",
            CoordinatorAction.Status.KILLED,
            "user1",
            "coord-app-name1",
            null,
            actualEnd,
            null);
    listener.onCoordinatorActionEvent(cae);
    summary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, "cj1@ca1");
    // check that all events are processed
    assertEquals(8, summary.getEventProcessed());
    assertEquals(EventStatus.END_MISS, summary.getEventStatus());
    assertEquals(3, slas.getSLACalculator().size());
  }