public void testWorkflowActionUpdate() throws Exception {
   WorkflowJobBean job =
       this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
   WorkflowActionBean action =
       addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);
   _testUpdateAction(action.getId());
 }
  public void testConfigNotPropagation() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set("abc", "xyz");
    workflow.setConf(newConf.toXmlString());

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");

    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });

    assertEquals(
        WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());

    subWorkflow.check(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.DONE, action.getStatus());

    subWorkflow.end(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
    assertNull(childConf.get("abc"));
  }
  /**
   * Test : verify the PreconditionException is thrown when actionCheckDelay > 0
   *
   * @throws Exception
   */
  public void testActionCheckPreCondition1() throws Exception {
    Instrumentation inst = Services.get().get(InstrumentationService.class).get();

    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean action =
        this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);

    ActionCheckXCommand checkCmd = new ActionCheckXCommand(action.getId(), 10);

    long counterVal;

    try {
      counterVal =
          inst.getCounters()
              .get(XCommand.INSTRUMENTATION_GROUP)
              .get(checkCmd.getName() + ".preconditionfailed")
              .getValue();
    } catch (NullPointerException e) {
      // counter might be null
      counterVal = 0L;
    }

    assertEquals(0L, counterVal);

    checkCmd.call();

    // precondition failed because of actionCheckDelay > 0
    counterVal =
        inst.getCounters()
            .get(XCommand.INSTRUMENTATION_GROUP)
            .get(checkCmd.getName() + ".preconditionfailed")
            .getValue();
    assertEquals(1L, counterVal);
  }
  public void testSubWorkflowConfCreation() throws Exception {
    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();

    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");

    WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
            + "      <app-path>hdfs://foo:9000/user/bar/workflow.xml</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");

    OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    assertNotNull(oozieClient);

    oozieClient =
        subWorkflow.getWorkflowClient(new Context(workflow, action), "http://localhost:8080/oozie");

    assertNotNull(oozieClient);
  }
  public void testActionCheck() throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean action =
        this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);
    WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(action.getId());

    new ActionStartXCommand(action.getId(), "map-reduce").call();
    action = jpaService.execute(wfActionGetCmd);

    ActionExecutorContext context =
        new ActionXCommand.ActionExecutorContext(job, action, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    JobConf conf =
        actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient =
        Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);

    String launcherId = action.getExternalId();

    final RunningJob launcherJob = jobClient.getJob(JobID.forName(launcherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
          }
        });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData =
        LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(action.getId()).call();
    action = jpaService.execute(wfActionGetCmd);
    String mapperId = action.getExternalId();
    String childId = action.getExternalChildIDs();

    assertTrue(launcherId.equals(mapperId));

    final RunningJob mrJob = jobClient.getJob(JobID.forName(childId));

    waitFor(
        120 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return mrJob.isComplete();
          }
        });
    assertTrue(mrJob.isSuccessful());

    new ActionCheckXCommand(action.getId()).call();
    action = jpaService.execute(wfActionGetCmd);

    assertEquals("SUCCEEDED", action.getExternalStatus());
  }
  public void testSubworkflowLib() throws Exception {
    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    FileSystem fs = getFileSystem();
    Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
    fs.create(parentLibJar);
    assertTrue(fs.exists(parentLibJar));
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
    workflow.setConf(newConf.toXmlString());

    Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();
    Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
    fs.create(subwfLibJar);
    assertTrue(fs.exists(subwfLibJar));

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "</sub-workflow>");
    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });

    assertEquals(
        WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
    subWorkflow.check(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.DONE, action.getStatus());
    subWorkflow.end(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
    childConf = wps.createProtoActionConf(childConf, "authToken", true);
    assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
  }
Esempio n. 7
0
 @Override
 protected void verifyPrecondition() throws CommandException, PreconditionException {
   if ((wfAction == null) || (wfAction.isComplete() && wfAction.isPending())) {
     if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
         && wfJob.getStatus() != WorkflowJob.Status.PREP) {
       throw new PreconditionException(ErrorCode.E0813, wfJob.getStatusStr());
     }
   } else {
     throw new PreconditionException(
         ErrorCode.E0814, actionId, wfAction.getStatusStr(), wfAction.isPending());
   }
 }
Esempio n. 8
0
  /**
   * Provides functionality to test kill node message
   *
   * @throws Exception
   */
  public void testKillNodeErrorMessage() throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-test-kill-node-message.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", "end.error");
    conf.set("external-status", "FAILED/KILLED");
    conf.set("signal-value", "fail");

    final String jobId = engine.submitJob(conf, true);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);

    waitFor(
        50000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
            return (job.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
          }
        });

    WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.KILLED, job.getStatus());

    WorkflowActionsGetForJobJPAExecutor wfActionsGetCmd =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actions = jpaService.execute(wfActionsGetCmd);

    int n = actions.size();
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
      if (bean.getType().equals("test")) {
        action = bean;
        break;
      }
    }
    assertNotNull(action);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals("end", action.getErrorMessage());
    assertEquals(WorkflowAction.Status.ERROR, action.getStatus());
  }
Esempio n. 9
0
  @SuppressWarnings("unchecked")
  public void testSetupMethodsForQuery() throws Exception {
    Hive2ActionExecutor ae = new Hive2ActionExecutor();
    List<Class> classes = new ArrayList<Class>();
    classes.add(Hive2Main.class);
    assertEquals(classes, ae.getLauncherClasses());

    String sampleQuery = "SELECT count(*) from foobar";
    Element actionXml =
        XmlUtils.parseXml(
            "<hive2  xmlns=\"uri:oozie:hive2-action:0.2\">"
                + "<job-tracker>"
                + getJobTrackerUri()
                + "</job-tracker>"
                + "<name-node>"
                + getNameNodeUri()
                + "</name-node>"
                + "<jdbc-url>jdbc:hive2://foo:1234/bar</jdbc-url>"
                + "<password>pass</password>"
                + "<query>"
                + sampleQuery
                + "</query>"
                + "<param>a=A</param>"
                + "<param>b=B</param>"
                + "<argument>-c</argument>"
                + "<argument>--dee</argument>"
                + "</hive2>");

    XConfiguration protoConf = new XConfiguration();
    protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());

    WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
    WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
    action.setType(ae.getType());

    Context context = new Context(wf, action);

    Configuration conf = ae.createBaseHadoopConf(context, actionXml);
    ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir());
    assertEquals("jdbc:hive2://foo:1234/bar", conf.get("oozie.hive2.jdbc.url"));
    assertEquals("pass", conf.get("oozie.hive2.password"));
    assertEquals(sampleQuery, conf.get("oozie.hive2.query"));
    assertNull(conf.get("oozie.hive2.script"));
    assertEquals("2", conf.get("oozie.hive2.params.size"));
    assertEquals("a=A", conf.get("oozie.hive2.params.0"));
    assertEquals("b=B", conf.get("oozie.hive2.params.1"));
    assertEquals("2", conf.get("oozie.hive2.args.size"));
    assertEquals("-c", conf.get("oozie.hive2.args.0"));
    assertEquals("--dee", conf.get("oozie.hive2.args.1"));
  }
  public void testExecuteUpdate() throws Exception {

    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean bean =
        addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);
    bean.setStatus(WorkflowAction.Status.RUNNING);
    bean.setName("test-name");
    WorkflowActionQueryExecutor.getInstance()
        .executeUpdate(WorkflowActionQuery.UPDATE_ACTION, bean);
    WorkflowActionBean retBean =
        WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION, bean.getId());
    assertEquals("test-name", retBean.getName());
    assertEquals(retBean.getStatus(), WorkflowAction.Status.RUNNING);
  }
Esempio n. 11
0
  private Context createContext(String actionXml) throws Exception {
    Hive2ActionExecutor ae = new Hive2ActionExecutor();

    XConfiguration protoConf = new XConfiguration();
    protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());

    SharelibUtils.addToDistributedCache("hive2", getFileSystem(), getFsTestCaseDir(), protoConf);

    WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
    WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
    action.setType(ae.getType());
    action.setConf(actionXml);

    return new Context(wf, action);
  }
Esempio n. 12
0
  /**
   * v1 service implementation to get a list of workflows, with filtering or interested windows
   * embedded in the request object
   */
  private JSONObject getWorkflowJobs(HttpServletRequest request) throws XServletException {
    JSONObject json = new JSONObject();
    try {
      String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
      String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
      String lenStr = request.getParameter(RestConstants.LEN_PARAM);
      int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
      start = (start < 1) ? 1 : start;
      int len = (lenStr != null) ? Integer.parseInt(lenStr) : 50;
      len = (len < 1) ? 50 : len;
      DagEngine dagEngine =
          Services.get()
              .get(DagEngineService.class)
              .getDagEngine(getUser(request), getAuthToken(request));
      WorkflowsInfo jobs = dagEngine.getJobs(filter, start, len);
      List<WorkflowJobBean> jsonWorkflows = jobs.getWorkflows();
      json.put(JsonTags.WORKFLOWS_JOBS, WorkflowJobBean.toJSONArray(jsonWorkflows));
      json.put(JsonTags.WORKFLOWS_TOTAL, jobs.getTotal());
      json.put(JsonTags.WORKFLOWS_OFFSET, jobs.getStart());
      json.put(JsonTags.WORKFLOWS_LEN, jobs.getLen());

    } catch (DagEngineException ex) {
      throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }

    return json;
  }
Esempio n. 13
0
  /**
   * This test case verifies if getRetryInterval picks up the overridden value.
   *
   * @throws Exception
   */
  public void testCheckInterval() throws Exception {
    long testedValue = 10;
    Services.get().getConf().setLong(ActionCheckerService.CONF_ACTION_CHECK_INTERVAL, testedValue);

    WorkflowJobBean job0 =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    final String jobId = job0.getId();
    WorkflowActionBean action0 =
        this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.RUNNING);
    final String actionId = action0.getId();

    ActionCheckXCommand checkCommand = new ActionCheckXCommand(actionId);
    checkCommand.call();
    long effectiveValue = checkCommand.getRetryInterval();
    assertEquals(testedValue, effectiveValue);
  }
 protected void updateSLASummary() throws XException {
   updateSLASummary(wfJob.getId(), wfJob.getStartTime(), wfJob.getEndTime(), wfJob.getStatusStr());
 }
Esempio n. 15
0
 /**
  * Check if the user+group is authorized to operate on the specified job.
  *
  * <p>Checks if the user is a super-user or the one who started the job.
  *
  * <p>Read operations are allowed to all users.
  *
  * @param user user name.
  * @param jobId job id.
  * @param write indicates if the check is for read or write job tasks.
  * @throws AuthorizationException thrown if the user is not authorized for the job.
  */
 public void authorizeForJob(String user, String jobId, boolean write)
     throws AuthorizationException {
   if (authorizationEnabled && write && !isAdmin(user)) {
     try {
       // handle workflow jobs
       if (jobId.endsWith("-W")) {
         WorkflowJobBean jobBean = null;
         JPAService jpaService = Services.get().get(JPAService.class);
         if (jpaService != null) {
           try {
             jobBean = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
           } catch (JPAExecutorException je) {
             throw new AuthorizationException(je);
           }
         } else {
           throw new AuthorizationException(ErrorCode.E0610);
         }
         if (jobBean != null && !jobBean.getUser().equals(user)) {
           if (!isUserInAcl(user, jobBean.getGroup())) {
             incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
             throw new AuthorizationException(ErrorCode.E0508, user, jobId);
           }
         }
       }
       // handle bundle jobs
       else if (jobId.endsWith("-B")) {
         BundleJobBean jobBean = null;
         JPAService jpaService = Services.get().get(JPAService.class);
         if (jpaService != null) {
           try {
             jobBean = jpaService.execute(new BundleJobGetJPAExecutor(jobId));
           } catch (JPAExecutorException je) {
             throw new AuthorizationException(je);
           }
         } else {
           throw new AuthorizationException(ErrorCode.E0610);
         }
         if (jobBean != null && !jobBean.getUser().equals(user)) {
           if (!isUserInAcl(user, jobBean.getGroup())) {
             incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
             throw new AuthorizationException(ErrorCode.E0509, user, jobId);
           }
         }
       }
       // handle coordinator jobs
       else {
         CoordinatorJobBean jobBean = null;
         JPAService jpaService = Services.get().get(JPAService.class);
         if (jpaService != null) {
           try {
             jobBean = jpaService.execute(new CoordJobGetJPAExecutor(jobId));
           } catch (JPAExecutorException je) {
             throw new AuthorizationException(je);
           }
         } else {
           throw new AuthorizationException(ErrorCode.E0610);
         }
         if (jobBean != null && !jobBean.getUser().equals(user)) {
           if (!isUserInAcl(user, jobBean.getGroup())) {
             incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
             throw new AuthorizationException(ErrorCode.E0509, user, jobId);
           }
         }
       }
     } catch (IOException ex) {
       throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex);
     }
   }
 }
Esempio n. 16
0
  /**
   * Provides functionality to test non transient failures and coordinator action update
   *
   * @param errorType the error type. (start.non-transient, end.non-transient)
   * @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
   * @param expErrorMsg expected error message.
   * @throws Exception
   */
  private void _testNonTransientWithCoordActionUpdate(
      String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);

    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());

    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);

    final String jobId = engine.submitJob(conf, false);

    final JPAService jpaService = Services.get().get(JPAService.class);
    final CoordinatorJobBean coordJob =
        addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, false, false);
    CoordinatorActionBean coordAction =
        addRecordToCoordActionTable(
            coordJob.getId(),
            1,
            CoordinatorAction.Status.RUNNING,
            "coord-action-get.xml",
            jobId,
            "RUNNING",
            0);

    engine.start(jobId);

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
          }
        });

    assertNotNull(jpaService);
    WorkflowJobGetJPAExecutor wfGetCmd = new WorkflowJobGetJPAExecutor(jobId);
    WorkflowJobBean job = jpaService.execute(wfGetCmd);
    WorkflowActionsGetForJobJPAExecutor actionsGetExe =
        new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actionsList = jpaService.execute(actionsGetExe);

    int n = actionsList.size();
    WorkflowActionBean action = actionsList.get(n - 1);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus1, action.getStatus());
    assertFalse(action.isPending());

    assertEquals(WorkflowJob.Status.SUSPENDED, job.getStatus());

    waitFor(
        5000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            CoordinatorActionBean coordAction2 =
                jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId));
            return coordAction2.getStatus().equals(CoordinatorAction.Status.SUSPENDED);
          }
        });

    coordAction = jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId));
    assertEquals(CoordinatorAction.Status.SUSPENDED, coordAction.getStatus());
  }
Esempio n. 17
0
  public void testActionCheckTransientDuringMRAction() throws Exception {
    // When using YARN, skip this test because it relies on shutting down the job tracker, which
    // isn't used in YARN
    if (createJobConf().get("yarn.resourcemanager.address") != null) {
      return;
    }
    services.destroy();
    // Make the max number of retries lower so the test won't take as long
    final int maxRetries = 2;
    setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
    services = new Services();
    // Disable ActionCheckerService so it doesn't interfere by triggering any extra
    // ActionCheckXCommands
    setClassesToBeExcluded(
        services.getConf(), new String[] {"org.apache.oozie.service.ActionCheckerService"});
    services.init();

    final JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job0 =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    final String jobId = job0.getId();
    WorkflowActionBean action0 =
        this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
    final String actionId = action0.getId();
    final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);

    new ActionStartXCommand(actionId, "map-reduce").call();
    final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
    String originalLauncherId = action1.getExternalId();

    ActionExecutorContext context =
        new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    JobConf conf =
        actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient =
        Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);

    final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
          }
        });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData =
        LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(action1.getId()).call();
    WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
    String originalMapperId = action2.getExternalChildIDs();

    assertFalse(originalLauncherId.equals(originalMapperId));

    // At this point, the launcher job has finished and the map-reduce action has started (but not
    // finished)
    // Now, shutdown the job tracker to pretend it has gone down during the map-reduce job
    executeWhileJobTrackerIsShutdown(
        new ShutdownJobTrackerExecutable() {
          @Override
          public void execute() throws Exception {
            assertEquals(0, action1.getRetries());
            new ActionCheckXCommand(actionId).call();

            waitFor(
                30 * 1000,
                new Predicate() {
                  @Override
                  public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() > 0);
                  }
                });
            waitFor(
                180 * 1000,
                new Predicate() {
                  @Override
                  public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() == 0);
                  }
                });
            WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
            assertEquals(0, action1b.getRetries());
            assertEquals("START_MANUAL", action1b.getStatusStr());

            WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
            assertEquals("SUSPENDED", job1.getStatusStr());

            // At this point, the action has gotten a transient error, even after maxRetries tries
            // so the workflow has been
            // SUSPENDED
          }
        });
    // Now, lets bring the job tracker back up and resume the workflow (which will restart the
    // current action)
    // It should now continue and finish with SUCCEEDED
    new ResumeXCommand(jobId).call();
    WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    assertEquals("RUNNING", job2.getStatusStr());

    sleep(500);

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
    String launcherId = action3.getExternalId();

    assertFalse(originalLauncherId.equals(launcherId));

    final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return launcherJob2.isComplete();
          }
        });

    assertTrue(launcherJob2.isSuccessful());
    actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherMapperHelper.hasIdSwap(actionData));

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
    String mapperId = action4.getExternalChildIDs();
    assertFalse(originalMapperId.equals(mapperId));

    final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));

    waitFor(
        120 * 1000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            return mrJob.isComplete();
          }
        });
    assertTrue(mrJob.isSuccessful());

    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);

    assertEquals("SUCCEEDED", action5.getExternalStatus());
  }
Esempio n. 18
0
  protected Void call(WorkflowStore store) throws StoreException, CommandException {
    WorkflowJobBean workflow = store.getWorkflow(jobId, false);
    setLogInfo(workflow);
    WorkflowActionBean action = store.getAction(id, false);
    setLogInfo(action);
    if (action.isPending()
        && (action.getStatus() == WorkflowActionBean.Status.DONE
            || action.getStatus() == WorkflowActionBean.Status.END_RETRY
            || action.getStatus() == WorkflowActionBean.Status.END_MANUAL)) {
      if (workflow.getStatus() == WorkflowJob.Status.RUNNING) {

        ActionExecutor executor =
            Services.get().get(ActionService.class).getExecutor(action.getType());
        Configuration conf = workflow.getWorkflowInstance().getConf();
        int maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries());
        long retryInterval =
            conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval());
        executor.setMaxRetries(maxRetries);
        executor.setRetryInterval(retryInterval);

        if (executor != null) {
          boolean isRetry = false;
          if (action.getStatus() == WorkflowActionBean.Status.END_RETRY
              || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) {
            isRetry = true;
          }
          ActionExecutorContext context =
              new ActionCommand.ActionExecutorContext(workflow, action, isRetry);
          try {

            XLog.getLog(getClass())
                .debug(
                    "End, name [{0}] type [{1}] status[{2}] external status [{3}] signal value [{4}]",
                    action.getName(),
                    action.getType(),
                    action.getStatus(),
                    action.getExternalStatus(),
                    action.getSignalValue());
            WorkflowInstance wfInstance = workflow.getWorkflowInstance();
            DagELFunctions.setActionInfo(wfInstance, action);
            workflow.setWorkflowInstance(wfInstance);
            incrActionCounter(action.getType(), 1);

            Instrumentation.Cron cron = new Instrumentation.Cron();
            cron.start();
            executor.end(context, action);
            cron.stop();
            addActionCron(action.getType(), cron);

            if (!context.isEnded()) {
              XLog.getLog(getClass())
                  .warn(
                      XLog.OPS,
                      "Action Ended, ActionExecutor [{0}] must call setEndData()",
                      executor.getType());
              action.setErrorInfo(
                  END_DATA_MISSING, "Execution Ended, but End Data Missing from Action");
              failJob(context);
              store.updateAction(action);
              store.updateWorkflow(workflow);
              return null;
            }
            action.setRetries(0);
            action.setEndTime(new Date());
            store.updateAction(action);
            store.updateWorkflow(workflow);
            Status slaStatus = null;
            switch (action.getStatus()) {
              case OK:
                slaStatus = Status.SUCCEEDED;
                break;
              case KILLED:
                slaStatus = Status.KILLED;
                break;
              case FAILED:
                slaStatus = Status.FAILED;
                break;
              case ERROR:
                XLog.getLog(getClass()).info("ERROR is considered as FAILED for SLA");
                slaStatus = Status.KILLED;
                break;
              default: // TODO: What will happen for other Action
                // status
                slaStatus = Status.FAILED;
                break;
            }
            SLADbOperations.writeStausEvent(
                action.getSlaXml(), action.getId(), store, slaStatus, SlaAppType.WORKFLOW_ACTION);
            queueCallable(new NotificationCommand(workflow, action));
            XLog.getLog(getClass())
                .debug(
                    "Queuing commands for action "
                        + id
                        + " status "
                        + action.getStatus()
                        + ", Set pending="
                        + action.getPending());
            queueCallable(new SignalCommand(workflow.getId(), id));
          } catch (ActionExecutorException ex) {
            XLog.getLog(getClass())
                .warn(
                    "Error ending action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]",
                    action.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage());
            action.setErrorInfo(ex.getErrorCode(), ex.getMessage());
            action.setEndTime(null);
            switch (ex.getErrorType()) {
              case TRANSIENT:
                if (!handleTransient(context, executor, WorkflowAction.Status.END_RETRY)) {
                  handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL);
                  action.setPendingAge(new Date());
                  action.setRetries(0);
                }
                action.setEndTime(null);
                break;
              case NON_TRANSIENT:
                handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL);
                action.setEndTime(null);
                break;
              case ERROR:
                handleError(context, executor, COULD_NOT_END, false, WorkflowAction.Status.ERROR);
                queueCallable(new SignalCommand(workflow.getId(), id));
                break;
              case FAILED:
                failJob(context);
                break;
            }
            store.updateAction(action);
            store.updateWorkflow(workflow);
          }
        } else {
          throw new CommandException(ErrorCode.E0802, action.getType());
        }
      } else {
        XLog.getLog(getClass())
            .warn(
                "Job state is not {0}. Skipping ActionEnd Execution",
                WorkflowJob.Status.RUNNING.toString());
      }
    } else {
      XLog.getLog(getClass())
          .debug(
              "Action pending={0}, status={1}. Skipping ActionEnd Execution",
              action.getPending(), action.getStatusStr());
    }
    return null;
  }
Esempio n. 19
0
  /**
   * Test : verify the PreconditionException is thrown when job != RUNNING && job != SUSPENDED
   *
   * @throws Exception
   */
  public void testActionCheckPreCondition4() throws Exception {
    Instrumentation inst = Services.get().get(InstrumentationService.class).get();

    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.FAILED, WorkflowInstance.Status.FAILED);
    WorkflowActionBean action =
        this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING);

    ActionCheckXCommand checkCmd = new ActionCheckXCommand(action.getId());

    long counterVal;

    try {
      counterVal =
          inst.getCounters()
              .get(XCommand.INSTRUMENTATION_GROUP)
              .get(checkCmd.getName() + ".preconditionfailed")
              .getValue();
    } catch (NullPointerException e) {
      // counter might be null
      counterVal = 0L;
    }

    assertEquals(0L, counterVal);

    checkCmd.call();

    // precondition failed because of job != RUNNING && job != SUSPENDED
    counterVal =
        inst.getCounters()
            .get(XCommand.INSTRUMENTATION_GROUP)
            .get(checkCmd.getName() + ".preconditionfailed")
            .getValue();
    assertEquals(1L, counterVal);

    job = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING);

    checkCmd = new ActionCheckXCommand(action.getId());

    checkCmd.call();

    // precondition passed because job == RUNNING so counter shouldn't have incremented
    counterVal =
        inst.getCounters()
            .get(XCommand.INSTRUMENTATION_GROUP)
            .get(checkCmd.getName() + ".preconditionfailed")
            .getValue();
    assertEquals(1L, counterVal);

    job =
        this.addRecordToWfJobTable(WorkflowJob.Status.SUSPENDED, WorkflowInstance.Status.SUSPENDED);
    action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING);

    checkCmd = new ActionCheckXCommand(action.getId());

    checkCmd.call();

    // precondition passed because job == SUSPENDED so counter shouldn't have incremented
    counterVal =
        inst.getCounters()
            .get(XCommand.INSTRUMENTATION_GROUP)
            .get(checkCmd.getName() + ".preconditionfailed")
            .getValue();
    assertEquals(1L, counterVal);

    job =
        this.addRecordToWfJobTable(WorkflowJob.Status.SUCCEEDED, WorkflowInstance.Status.SUCCEEDED);
    action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING);

    checkCmd = new ActionCheckXCommand(action.getId());

    checkCmd.call();

    // precondition failed because of job != RUNNING && job != SUSPENDED
    counterVal =
        inst.getCounters()
            .get(XCommand.INSTRUMENTATION_GROUP)
            .get(checkCmd.getName() + ".preconditionfailed")
            .getValue();
    assertEquals(2L, counterVal);

    job = this.addRecordToWfJobTable(WorkflowJob.Status.KILLED, WorkflowInstance.Status.KILLED);
    action = this.addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.RUNNING);

    checkCmd = new ActionCheckXCommand(action.getId());

    checkCmd.call();

    // precondition failed because of job != RUNNING && job != SUSPENDED
    counterVal =
        inst.getCounters()
            .get(XCommand.INSTRUMENTATION_GROUP)
            .get(checkCmd.getName() + ".preconditionfailed")
            .getValue();
    assertEquals(3L, counterVal);
  }
  public void testGetUpdateQuery() throws Exception {
    EntityManager em = jpaService.getEntityManager();
    WorkflowJobBean job =
        this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    WorkflowActionBean bean =
        addRecordToWfActionTable(job.getId(), "1", WorkflowAction.Status.PREP);

    // UPDATE_ACTION
    Query query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em);
    assertEquals(query.getParameterValue("conf"), bean.getConfBlob());
    assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("externalId"), bean.getExternalId());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("name"), bean.getName());
    assertEquals(query.getParameterValue("cred"), bean.getCred());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("type"), bean.getType());
    assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp());
    assertEquals(query.getParameterValue("executionPath"), bean.getExecutionPath());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("logToken"), bean.getLogToken());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAge());
    assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue());
    assertEquals(query.getParameterValue("slaXml"), bean.getSlaXmlBlob());
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("wfId"), bean.getWfId());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_FOR_LAST_CHECKED_TIME
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION, bean, em);
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_PENDING
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("id"), bean.getId());

    // UPDATE_ACTION_STATUS_PENDING
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_PENDING_TRANS
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_PENDING_TRANS_ERROR
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS_ERROR, bean, em);
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("transition"), bean.getTransition());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_START
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_START, bean, em);
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("conf"), bean.getConfBlob());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("externalId"), bean.getExternalId());
    assertEquals(query.getParameterValue("trackerUri"), bean.getTrackerUri());
    assertEquals(query.getParameterValue("consoleUrl"), bean.getConsoleUrl());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_CHECK
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_CHECK, bean, em);
    assertEquals(query.getParameterValue("externalChildIDs"), bean.getExternalChildIDsBlob());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("data"), bean.getDataBlob());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("lastCheckTime"), bean.getLastCheckTimestamp());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("startTime"), bean.getStartTimestamp());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("id"), bean.getId());
    // UPDATE_ACTION_END
    query =
        WorkflowActionQueryExecutor.getInstance()
            .getUpdateQuery(WorkflowActionQuery.UPDATE_ACTION_END, bean, em);
    assertEquals(query.getParameterValue("errorCode"), bean.getErrorCode());
    assertEquals(query.getParameterValue("errorMessage"), bean.getErrorMessage());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("endTime"), bean.getEndTimestamp());
    assertEquals(query.getParameterValue("status"), bean.getStatus().toString());
    assertEquals(query.getParameterValue("retries"), bean.getRetries());
    assertEquals(query.getParameterValue("pending"), bean.getPending());
    assertEquals(query.getParameterValue("pendingAge"), bean.getPendingAgeTimestamp());
    assertEquals(query.getParameterValue("signalValue"), bean.getSignalValue());
    assertEquals(query.getParameterValue("userRetryCount"), bean.getUserRetryCount());
    assertEquals(query.getParameterValue("externalStatus"), bean.getExternalStatus());
    assertEquals(query.getParameterValue("stats"), bean.getStatsBlob());
    assertEquals(query.getParameterValue("id"), bean.getId());
    em.close();
  }
  public void testSubWorkflowRecovery() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");

    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });
    String extId = action.getExternalId();
    assertEquals(WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(extId).getStatus());
    WorkflowActionBean action1 = new WorkflowActionBean();
    action1.setId(action.getId());
    action1.setName(action.getName());
    action1.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
            + "      <app-path>wrongAppPath</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");
    subWorkflow.start(new Context(workflow, action1), action1);
    assertEquals(extId, action1.getExternalId());
    subWorkflow.check(new Context(workflow, action1), action1);
    assertEquals(WorkflowAction.Status.DONE, action1.getStatus());
    subWorkflow.end(new Context(workflow, action1), action1);
    assertEquals(WorkflowAction.Status.OK, action1.getStatus());
  }
Esempio n. 22
0
  @Override
  protected Void execute() throws CommandException {
    LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
    WorkflowInstance workflowInstance = wfJob.getWorkflowInstance();
    workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob);
    boolean completed = false;
    boolean skipAction = false;
    if (wfAction == null) {
      if (wfJob.getStatus() == WorkflowJob.Status.PREP) {
        try {
          completed = workflowInstance.start();
        } catch (WorkflowException e) {
          throw new CommandException(e);
        }
        wfJob.setStatus(WorkflowJob.Status.RUNNING);
        wfJob.setStartTime(new Date());
        wfJob.setWorkflowInstance(workflowInstance);
        // 1. Add SLA status event for WF-JOB with status STARTED
        // 2. Add SLA registration events for all WF_ACTIONS
        SLADbXOperations.writeStausEvent(
            wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB);
        writeSLARegistrationForAllActions(
            workflowInstance.getApp().getDefinition(),
            wfJob.getUser(),
            wfJob.getGroup(),
            wfJob.getConf());
        queue(new NotificationXCommand(wfJob));
      } else {
        throw new CommandException(ErrorCode.E0801, wfJob.getId());
      }
    } else {
      String skipVar =
          workflowInstance.getVar(
              wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
      if (skipVar != null) {
        skipAction = skipVar.equals("true");
      }
      try {
        completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue());
      } catch (WorkflowException e) {
        throw new CommandException(e);
      }
      wfJob.setWorkflowInstance(workflowInstance);
      wfAction.resetPending();
      if (!skipAction) {
        wfAction.setTransition(workflowInstance.getTransition(wfAction.getName()));
      }
      try {
        jpaService.execute(new WorkflowActionUpdateJPAExecutor(wfAction));
      } catch (JPAExecutorException je) {
        throw new CommandException(je);
      }
    }

    if (completed) {
      try {
        for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) {
          WorkflowActionBean actionToKill;

          actionToKill = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToKillId));

          actionToKill.setPending();
          actionToKill.setStatus(WorkflowActionBean.Status.KILLED);
          jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToKill));
          queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType()));
        }

        for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) {
          WorkflowActionBean actionToFail =
              jpaService.execute(new WorkflowActionGetJPAExecutor(actionToFailId));
          actionToFail.resetPending();
          actionToFail.setStatus(WorkflowActionBean.Status.FAILED);
          SLADbXOperations.writeStausEvent(
              wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
          jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToFail));
        }
      } catch (JPAExecutorException je) {
        throw new CommandException(je);
      }

      wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
      wfJob.setEndTime(new Date());
      wfJob.setWorkflowInstance(workflowInstance);
      Status slaStatus = Status.SUCCEEDED;
      switch (wfJob.getStatus()) {
        case SUCCEEDED:
          slaStatus = Status.SUCCEEDED;
          break;
        case KILLED:
          slaStatus = Status.KILLED;
          break;
        case FAILED:
          slaStatus = Status.FAILED;
          break;
        default: // TODO SUSPENDED
          break;
      }
      SLADbXOperations.writeStausEvent(
          wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB);
      queue(new NotificationXCommand(wfJob));
      if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) {
        InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation());
      }
    } else {
      for (WorkflowActionBean newAction :
          WorkflowStoreService.getStartedActions(workflowInstance)) {
        String skipVar =
            workflowInstance.getVar(
                newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
        boolean skipNewAction = false;
        if (skipVar != null) {
          skipNewAction = skipVar.equals("true");
        }
        try {
          if (skipNewAction) {
            WorkflowActionBean oldAction;

            oldAction = jpaService.execute(new WorkflowActionGetJPAExecutor(newAction.getId()));

            oldAction.setPending();
            jpaService.execute(new WorkflowActionUpdateJPAExecutor(oldAction));

            queue(new SignalXCommand(jobId, oldAction.getId()));
          } else {
            newAction.setPending();
            String actionSlaXml =
                getActionSLAXml(
                    newAction.getName(),
                    workflowInstance.getApp().getDefinition(),
                    wfJob.getConf());
            newAction.setSlaXml(actionSlaXml);
            jpaService.execute(new WorkflowActionInsertJPAExecutor(newAction));
            LOG.debug(
                "SignalXCommand: Name: "
                    + newAction.getName()
                    + ", Id: "
                    + newAction.getId()
                    + ", Authcode:"
                    + newAction.getCred());
            queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
          }
        } catch (JPAExecutorException je) {
          throw new CommandException(je);
        }
      }
    }

    try {
      jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob));
    } catch (JPAExecutorException je) {
      throw new CommandException(je);
    }
    XLog.getLog(getClass())
        .debug(
            "Updated the workflow status to "
                + wfJob.getId()
                + "  status ="
                + wfJob.getStatusStr());
    if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
        && wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) {
      // update coordinator action
      new CoordActionUpdateXCommand(wfJob).call();
      new WfEndXCommand(wfJob).call(); // To delete the WF temp dir
    }
    LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
    return null;
  }
  public void testGetGroupFromParent() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    String actionConf =
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>";

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(actionConf);

    // negative test
    final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    workflow.setConf(newConf.toXmlString());

    subWorkflow.start(new Context(workflow, action), action);

    OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        5000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            subWorkflow.check(new Context(workflow, action), action);
            return action.getStatus() == WorkflowActionBean.Status.DONE;
          }
        });

    subWorkflow.check(new Context(workflow, action), action);
    subWorkflow.end(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));

    assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME));

    // positive test
    newConf.set(OozieClient.GROUP_NAME, getTestGroup());
    workflow.setConf(newConf.toXmlString());
    final WorkflowActionBean action1 = new WorkflowActionBean();
    action1.setConf(actionConf);
    action1.setId("W1");

    subWorkflow.start(new Context(workflow, action1), action1);

    oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL);

    waitFor(
        5000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            subWorkflow.check(new Context(workflow, action1), action1);
            return action1.getStatus() == WorkflowActionBean.Status.DONE;
          }
        });

    subWorkflow.check(new Context(workflow, action1), action1);
    subWorkflow.end(new Context(workflow, action1), action1);

    wf = oozieClient.getJobInfo(action1.getExternalId());
    childConf = new XConfiguration(new StringReader(wf.getConf()));
    assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME));
  }
 @Override
 protected boolean isJobEnded() {
   return wfJob.inTerminalState();
 }