Esempio n. 1
0
 public static WorkflowJob.Status killedOozieJob(String oozieJobid) throws OozieClientException {
   OozieClient wc = new AuthOozieClient("http://10.25.22.19:8080/oozie", "ldap");
   wc.setHeader("j_username", "V_PA011_HADOOP_CORE");
   wc.setHeader("j_password", "{DES}/+J5e0Ed2/PABTqzeVGQ1g==");
   wc.kill(oozieJobid);
   return wc.getJobInfo(oozieJobid).getStatus();
 }
  public void testConfigNotPropagation() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set("abc", "xyz");
    workflow.setConf(newConf.toXmlString());

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");

    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });

    assertEquals(
        WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());

    subWorkflow.check(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.DONE, action.getStatus());

    subWorkflow.end(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
    assertNull(childConf.get("abc"));
  }
Esempio n. 3
0
  /**
   * Run recipe with different frequencies. Submission should go through. Check frequency of the
   * launched oozie job
   */
  @Test(dataProvider = "frequencyGenerator")
  public void differentRecipeFrequenciesTest(String frequency) throws Exception {
    setUp(RecipeExecLocation.SourceCluster);
    LOGGER.info("Testing with frequency: " + frequency);
    String tblName = "myTable";
    recipeMerlin
        .withSourceDb(DB_NAME)
        .withSourceTable(tblName)
        .withFrequency(new Frequency(frequency));
    runSql(connection, "create table " + tblName + "(comment string)");
    final List<String> command = recipeMerlin.getSubmissionCommand();
    Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
    LOGGER.info("Submission went through.");

    InstanceUtil.waitTillInstanceReachState(
        clusterOC, recipeMerlin.getName(), 1, CoordinatorAction.Status.RUNNING, EntityType.PROCESS);
    String filter = "name=FALCON_PROCESS_" + recipeMerlin.getName();
    List<BundleJob> bundleJobs = OozieUtil.getBundles(clusterOC, filter, 0, 10);
    List<String> bundleIds = OozieUtil.getBundleIds(bundleJobs);
    String bundleId = OozieUtil.getMaxId(bundleIds);
    List<CoordinatorJob> coords = clusterOC.getBundleJobInfo(bundleId).getCoordinators();
    List<String> cIds = new ArrayList<String>();
    for (CoordinatorJob coord : coords) {
      cIds.add(coord.getId());
    }
    String coordId = OozieUtil.getMinId(cIds);
    CoordinatorJob job = clusterOC.getCoordJobInfo(coordId);
    CoordinatorJob.Timeunit timeUnit = job.getTimeUnit();
    String freq = job.getFrequency();
    LOGGER.info("Frequency of running job: " + timeUnit + " " + freq);
    Assert.assertTrue(
        frequency.contains(timeUnit.name().toLowerCase().replace("_", ""))
            && frequency.contains(freq),
        "Running job has different frequency.");
  }
  public void testSubworkflowLib() throws Exception {
    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    FileSystem fs = getFileSystem();
    Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
    fs.create(parentLibJar);
    assertTrue(fs.exists(parentLibJar));
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
    workflow.setConf(newConf.toXmlString());

    Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();
    Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
    fs.create(subwfLibJar);
    assertTrue(fs.exists(subwfLibJar));

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "</sub-workflow>");
    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });

    assertEquals(
        WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
    subWorkflow.check(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.DONE, action.getStatus());
    subWorkflow.end(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
    childConf = wps.createProtoActionConf(childConf, "authToken", true);
    assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
  }
Esempio n. 5
0
 private void copyOozieLog(OozieClient client, FileSystem fs, Path path, String id)
     throws OozieClientException, IOException {
   InputStream in = new ByteArrayInputStream(client.getJobLog(id).getBytes());
   OutputStream out = fs.create(new Path(path, "oozie.log"));
   IOUtils.copyBytes(in, out, 4096, true);
   LOG.info("Copied oozie log to {}", path);
 }
 /*
  * Retrieves replication coordinator instances.
  * @param client target oozie client
  * @param fName feed name
  */
 private List<CoordinatorAction> getReplicationInstances(OozieClient client, String fName)
     throws OozieClientException {
   String filter = "name=FALCON_FEED_" + fName;
   List<BundleJob> bundleJobs = OozieUtil.getBundles(client, filter, 0, 10);
   Assert.assertNotEquals(bundleJobs.size(), 0, "Could not retrieve bundles");
   List<String> bundleIds = OozieUtil.getBundleIds(bundleJobs);
   String bundleId = OozieUtil.getMaxId(bundleIds);
   LOGGER.info(String.format("Using bundle %s", bundleId));
   List<CoordinatorJob> coords = client.getBundleJobInfo(bundleId).getCoordinators();
   String coordId = null;
   for (CoordinatorJob coord : coords) {
     if (coord.getAppName().contains("FEED_REPLICATION")) {
       coordId = coord.getId();
       break;
     }
   }
   LOGGER.info(String.format("Using coordinator id: %s", coordId));
   Assert.assertNotNull(coordId, "Replication coordinator not found.");
   CoordinatorJob coordinatorJob = client.getCoordJobInfo(coordId);
   return coordinatorJob.getActions();
 }
  @Test
  public void testGetValidationWarnings_incompatibleVersions() throws Exception {
    OozieJobExecutorConfig config = new OozieJobExecutorConfig();

    OozieClient client = getBadConfigTest();
    OozieJobExecutorJobEntry je =
        new TestOozieJobExecutorJobEntry(client); // just use this to force an error condition

    config.setOozieUrl("http://localhost/oozie");
    config.setOozieWorkflowConfig("file:///test/job.properties");
    config.setJobEntryName("name");

    List<String> warnings = je.getValidationWarnings(config);

    assertEquals(1, warnings.size());
    assertEquals(
        BaseMessages.getString(
            OozieJobExecutorJobEntry.class,
            "ValidationMessages.Incompatible.Oozie.Versions",
            client.getClientBuildVersion()),
        warnings.get(0));
  }
Esempio n. 8
0
  public static WorkflowJob.Status queryJobStatus(String oozieJobid)
      throws OozieClientException, InterruptedException {
    OozieClient wc = new AuthOozieClient("http://10.25.22.19:8080/oozie", "ldap");
    wc.setHeader("j_username", "V_PA011_HADOOP_CORE");
    wc.setHeader("j_password", "{DES}/+J5e0Ed2/PABTqzeVGQ1g==");

    List<WorkflowAction> walist;
    int wacount = 0;
    while (wacount < 2) {
      walist = wc.getJobInfo(oozieJobid).getActions();
      wacount = walist.size();
      Thread.sleep(20000);
      System.out.println("#######-" + wacount);
      for (WorkflowAction wa : walist) {
        System.out.println("#######" + wa.getName());

        if (wa.getConsoleUrl().contains("jobdetails.jsp?jobid=")) {
          System.out.println("#######" + wa.getConsoleUrl());
        }
      }
    }

    return wc.getJobInfo(oozieJobid).getStatus();
  }
Esempio n. 9
0
  public static String oozieSubmitJob() throws Exception {
    return OozieClient.doAs(
        "hadoop",
        new Callable<String>() {
          @Override
          public String call() throws Exception {
            OozieClient wc = new AuthOozieClient("http://10.25.22.19:8080/oozie", "ldap");
            Properties conf = wc.createConfiguration();
            // setting workflow parameters
            // wc.doAs("hadoop");
            wc.setHeader("j_username", "V_PA011_HADOOP_CORE");
            wc.setHeader("j_password", "{DES}/+J5e0Ed2/PABTqzeVGQ1g==");

            conf.setProperty(OozieClient.USER_NAME, "hadoop");
            conf.setProperty(OozieClient.GROUP_NAME, "hadoop");
            conf.setProperty(
                OozieClient.APP_PATH, "hdfs://10.25.22.19:49000/apps/actionTest/shellaction");
            conf.setProperty("namenode_address", "hdfs://10.25.22.19:49000");
            conf.setProperty("jobtracker_address", "10.25.22.19:49001");
            conf.setProperty("mapred_job_queue_name", "queue01");

            conf.setProperty("data_db_username", "PSCPHDP");
            conf.setProperty("data_db_password", "{DES}4ueyVO2J9+v8ML7FGs608g==");
            conf.setProperty("data_db_url", "jdbc:oracle:thin:@10.25.18.28:1582:pa18dvp");
            conf.setProperty("nominalformateDate", "2013-04-12");
            conf.setProperty("oozie.use.system.libpath", "true");
            String jobId = wc.run(conf);
            System.out.println(jobId);
            // wait until the workflo w job finishes printing the status every 10 seconds
            //        	    while (wc.getJobInfo(jobId).getStatus() == WorkflowJob.Status.RUNNING) {
            //        	        System.out.println("Workflow job running ...");
            //        	        Thread.sleep(10 * 1000);
            //        	    }
            // print the final status o the workflow job
            System.out.println("Workflow job completed ...");
            System.out.println(wc.getJobInfo(jobId));
            return jobId;
          }
        });
  }
Esempio n. 10
0
  public void testFsFailover() throws Exception {
    Reader reader = IOUtils.getResourceAsReader("failover-fs-wf.xml", -1);
    Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
    IOUtils.copyCharStream(reader, writer);

    final OozieClient wfClient = LocalOozie.getClient();
    Properties conf = wfClient.createConfiguration();
    conf.setProperty(OozieClient.APP_PATH, getTestCaseDir());
    conf.setProperty(OozieClient.USER_NAME, getTestUser());
    conf.setProperty(OozieClient.GROUP_NAME, getTestGroup());
    injectKerberosInfo(conf);
    final Path source = new Path(getFsTestCaseDir(), "fsfailover-source");
    getFileSystem().mkdirs(source);
    final Path target = new Path(getFsTestCaseDir().toString(), "fsfailover-target");
    conf.setProperty("source", source.toString());
    conf.setProperty("target", target.toUri().getPath());

    setSystemProperty(FaultInjection.FAULT_INJECTION, "true");
    setSystemProperty(SkipCommitFaultInjection.ACTION_FAILOVER_FAULT_INJECTION, "true");

    final String jobId1 = wfClient.submit(conf);
    wfClient.start(jobId1);

    waitFor(
        10 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return getFileSystem().exists(target);
          }
        });
    assertTrue(getFileSystem().exists(target));

    waitFor(
        10 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection");
          }
        });
    assertFalse(FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection"));

    assertEquals(WorkflowJob.Status.RUNNING, wfClient.getJobInfo(jobId1).getStatus());

    WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();

    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId1, false);
    WorkflowActionBean action = actions.get(0);
    assertEquals(WorkflowAction.Status.PREP, action.getStatus());

    setSystemProperty(FaultInjection.FAULT_INJECTION, "false");
    setSystemProperty(SkipCommitFaultInjection.ACTION_FAILOVER_FAULT_INJECTION, "false");

    ActionStartCommand actionStartCommand =
        new ActionStartCommand(action.getId(), action.getType());
    actionStartCommand.call();

    store = Services.get().get(WorkflowStoreService.class).create();
    actions = store.getActionsForWorkflow(jobId1, false);
    action = actions.get(0);
    assertEquals(WorkflowAction.Status.DONE, action.getStatus());

    waitFor(
        5 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.SUCCEEDED;
          }
        });
    assertEquals(WorkflowJob.Status.SUCCEEDED, wfClient.getJobInfo(jobId1).getStatus());

    final String jobId2 = wfClient.submit(conf);

    wfClient.start(jobId2);
    waitFor(
        10 * 1000,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return wfClient.getJobInfo(jobId2).getStatus() == WorkflowJob.Status.KILLED;
          }
        });
    assertEquals(WorkflowJob.Status.KILLED, wfClient.getJobInfo(jobId2).getStatus());
  }
  public void testGetGroupFromParent() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    String actionConf =
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>";

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(actionConf);

    // negative test
    final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    workflow.setConf(newConf.toXmlString());

    subWorkflow.start(new Context(workflow, action), action);

    OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        5000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            subWorkflow.check(new Context(workflow, action), action);
            return action.getStatus() == WorkflowActionBean.Status.DONE;
          }
        });

    subWorkflow.check(new Context(workflow, action), action);
    subWorkflow.end(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));

    assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME));

    // positive test
    newConf.set(OozieClient.GROUP_NAME, getTestGroup());
    workflow.setConf(newConf.toXmlString());
    final WorkflowActionBean action1 = new WorkflowActionBean();
    action1.setConf(actionConf);
    action1.setId("W1");

    subWorkflow.start(new Context(workflow, action1), action1);

    oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL);

    waitFor(
        5000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            subWorkflow.check(new Context(workflow, action1), action1);
            return action1.getStatus() == WorkflowActionBean.Status.DONE;
          }
        });

    subWorkflow.check(new Context(workflow, action1), action1);
    subWorkflow.end(new Context(workflow, action1), action1);

    wf = oozieClient.getJobInfo(action1.getExternalId());
    childConf = new XConfiguration(new StringReader(wf.getConf()));
    assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME));
  }
  public void testSubWorkflowRecovery() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");

    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });
    String extId = action.getExternalId();
    assertEquals(WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(extId).getStatus());
    WorkflowActionBean action1 = new WorkflowActionBean();
    action1.setId(action.getId());
    action1.setName(action.getName());
    action1.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1'>"
            + "      <app-path>wrongAppPath</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");
    subWorkflow.start(new Context(workflow, action1), action1);
    assertEquals(extId, action1.getExternalId());
    subWorkflow.check(new Context(workflow, action1), action1);
    assertEquals(WorkflowAction.Status.DONE, action1.getStatus());
    subWorkflow.end(new Context(workflow, action1), action1);
    assertEquals(WorkflowAction.Status.OK, action1.getStatus());
  }
Esempio n. 13
0
  public int run(WorkflowExecutionContext context) {
    try {
      OozieClient client = new OozieClient(context.getWorkflowEngineUrl());
      WorkflowJob jobInfo;
      try {
        jobInfo = client.getJobInfo(context.getUserSubflowId());
      } catch (OozieClientException e) {
        LOG.error("Error getting jobinfo for: {}", context.getUserSubflowId(), e);
        return 0;
      }

      // Assumption is - Each wf run will have a directory
      // the corresponding job logs are stored within the respective dir
      Path path =
          new Path(context.getLogDir() + "/" + String.format("%03d", context.getWorkflowRunId()));
      FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri(), getConf());

      if (EntityType.FEED.name().equalsIgnoreCase(context.getEntityType())
          || notUserWorkflowEngineIsOozie(context.getUserWorkflowEngine())) {
        // if replication wf, retention wf or PIG Process
        copyOozieLog(client, fs, path, jobInfo.getId());

        List<WorkflowAction> workflowActions = jobInfo.getActions();
        for (int i = 0; i < workflowActions.size(); i++) {
          if (FALCON_ACTIONS.contains(workflowActions.get(i).getName())) {
            copyTTlogs(fs, path, jobInfo.getActions().get(i));
            break;
          }
        }
      } else {
        String flowId;
        // if process wf with pig, hive
        if (context.getUserWorkflowEngine().equals("pig")
            || context.getUserWorkflowEngine().equals("hive")) {
          flowId = jobInfo.getId();
        } else {
          // if process wf with oozie engine
          flowId = jobInfo.getExternalId();
        }
        copyOozieLog(client, fs, path, flowId);
        WorkflowJob subflowInfo = client.getJobInfo(flowId);
        List<WorkflowAction> actions = subflowInfo.getActions();
        for (WorkflowAction action : actions) {
          if (isActionTypeSupported(action)) {
            LOG.info(
                "Copying hadoop TT log for action: {} of type: {}",
                action.getName(),
                action.getType());
            copyTTlogs(fs, path, action);
          } else {
            LOG.info(
                "Ignoring hadoop TT log for non supported action: {} of type: {}",
                action.getName(),
                action.getType());
          }
        }
      }

    } catch (Exception e) {
      // JobLogMover doesn't throw exception, a failed log mover will not fail the user workflow
      LOG.error("Exception in log mover:", e);
    }
    return 0;
  }