コード例 #1
0
  private RunningJob submitAction(Context context, Namespace ns) throws Exception {
    Hive2ActionExecutor ae = new Hive2ActionExecutor();

    WorkflowAction action = context.getAction();

    ae.prepareActionDir(getFileSystem(), context);
    ae.submitLauncher(getFileSystem(), context, action);

    String jobId = action.getExternalId();
    String jobTracker = action.getTrackerUri();
    String consoleUrl = action.getConsoleUrl();
    assertNotNull(jobId);
    assertNotNull(jobTracker);
    assertNotNull(consoleUrl);
    Element e = XmlUtils.parseXml(action.getConf());
    XConfiguration conf =
        new XConfiguration(
            new StringReader(XmlUtils.prettyPrint(e.getChild("configuration", ns)).toString()));
    conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker", ns));
    conf.set("fs.default.name", e.getChildTextTrim("name-node", ns));
    conf.set("user.name", context.getProtoActionConf().get("user.name"));
    conf.set("group.name", getTestGroup());

    JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
    XConfiguration.copy(conf, jobConf);
    String user = jobConf.get("user.name");
    JobClient jobClient =
        Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf);
    final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
    assertNotNull(runningJob);
    return runningJob;
  }
コード例 #2
0
  public void testConfigNotPropagation() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set("abc", "xyz");
    workflow.setConf(newConf.toXmlString());

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>");

    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });

    assertEquals(
        WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());

    subWorkflow.check(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.DONE, action.getStatus());

    subWorkflow.end(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
    assertNull(childConf.get("abc"));
  }
コード例 #3
0
 /**
  * Returns a new XConfiguration instance with all inline values resolved.
  *
  * @return a new XConfiguration instance with all inline values resolved.
  */
 public XConfiguration resolve() {
   XConfiguration resolved = new XConfiguration();
   for (Map.Entry<String, String> entry : this) {
     resolved.set(entry.getKey(), get(entry.getKey()));
   }
   return resolved;
 }
コード例 #4
0
 /**
  * Returns a new XConfiguration with all values trimmed.
  *
  * @return a new XConfiguration with all values trimmed.
  */
 public XConfiguration trim() {
   XConfiguration trimmed = new XConfiguration();
   for (Map.Entry<String, String> entry : this) {
     trimmed.set(entry.getKey(), entry.getValue().trim());
   }
   return trimmed;
 }
コード例 #5
0
  public void testSubworkflowLib() throws Exception {
    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    FileSystem fs = getFileSystem();
    Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
    fs.create(parentLibJar);
    assertTrue(fs.exists(parentLibJar));
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
    workflow.setConf(newConf.toXmlString());

    Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();
    Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
    fs.create(subwfLibJar);
    assertTrue(fs.exists(subwfLibJar));

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "</sub-workflow>");
    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);

    final OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        JOB_TIMEOUT,
        new Predicate() {
          public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus()
                == WorkflowJob.Status.SUCCEEDED;
          }
        });

    assertEquals(
        WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
    subWorkflow.check(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.DONE, action.getStatus());
    subWorkflow.end(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
    childConf = wps.createProtoActionConf(childConf, "authToken", true);
    assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
  }
コード例 #6
0
 /**
  * Return a configuration with all sensitive values masked.
  *
  * @return masked configuration.
  */
 public Configuration getMaskedConfiguration() {
   XConfiguration maskedConf = new XConfiguration();
   Configuration conf = getConf();
   for (Map.Entry<String, String> entry : conf) {
     String name = entry.getKey();
     String value = getValue(conf, name);
     maskedConf.set(name, value);
   }
   return maskedConf;
 }
コード例 #7
0
  @SuppressWarnings("unchecked")
  public void testSetupMethodsForQuery() throws Exception {
    Hive2ActionExecutor ae = new Hive2ActionExecutor();
    List<Class> classes = new ArrayList<Class>();
    classes.add(Hive2Main.class);
    assertEquals(classes, ae.getLauncherClasses());

    String sampleQuery = "SELECT count(*) from foobar";
    Element actionXml =
        XmlUtils.parseXml(
            "<hive2  xmlns=\"uri:oozie:hive2-action:0.2\">"
                + "<job-tracker>"
                + getJobTrackerUri()
                + "</job-tracker>"
                + "<name-node>"
                + getNameNodeUri()
                + "</name-node>"
                + "<jdbc-url>jdbc:hive2://foo:1234/bar</jdbc-url>"
                + "<password>pass</password>"
                + "<query>"
                + sampleQuery
                + "</query>"
                + "<param>a=A</param>"
                + "<param>b=B</param>"
                + "<argument>-c</argument>"
                + "<argument>--dee</argument>"
                + "</hive2>");

    XConfiguration protoConf = new XConfiguration();
    protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());

    WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
    WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
    action.setType(ae.getType());

    Context context = new Context(wf, action);

    Configuration conf = ae.createBaseHadoopConf(context, actionXml);
    ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir());
    assertEquals("jdbc:hive2://foo:1234/bar", conf.get("oozie.hive2.jdbc.url"));
    assertEquals("pass", conf.get("oozie.hive2.password"));
    assertEquals(sampleQuery, conf.get("oozie.hive2.query"));
    assertNull(conf.get("oozie.hive2.script"));
    assertEquals("2", conf.get("oozie.hive2.params.size"));
    assertEquals("a=A", conf.get("oozie.hive2.params.0"));
    assertEquals("b=B", conf.get("oozie.hive2.params.1"));
    assertEquals("2", conf.get("oozie.hive2.args.size"));
    assertEquals("-c", conf.get("oozie.hive2.args.0"));
    assertEquals("--dee", conf.get("oozie.hive2.args.1"));
  }
コード例 #8
0
  private Context createContext(String actionXml) throws Exception {
    Hive2ActionExecutor ae = new Hive2ActionExecutor();

    XConfiguration protoConf = new XConfiguration();
    protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());

    SharelibUtils.addToDistributedCache("hive2", getFileSystem(), getFsTestCaseDir(), protoConf);

    WorkflowJobBean wf = createBaseWorkflow(protoConf, "hive2-action");
    WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
    action.setType(ae.getType());
    action.setConf(actionXml);

    return new Context(wf, action);
  }
コード例 #9
0
ファイル: CoordSubmitXCommand.java プロジェクト: kirann/oozie
  /**
   * Merge default configuration with user-defined configuration.
   *
   * @throws CommandException thrown if failed to read or merge configurations
   */
  protected void mergeDefaultConfig() throws CommandException {
    Path configDefault = null;
    try {
      String coordAppPathStr = conf.get(OozieClient.COORDINATOR_APP_PATH);
      Path coordAppPath = new Path(coordAppPathStr);
      String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
      String group =
          ParamChecker.notEmpty(conf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
      FileSystem fs =
          Services.get()
              .get(HadoopAccessorService.class)
              .createFileSystem(user, group, coordAppPath.toUri(), new Configuration());

      // app path could be a directory
      if (!fs.isFile(coordAppPath)) {
        configDefault = new Path(coordAppPath, CONFIG_DEFAULT);
      } else {
        configDefault = new Path(coordAppPath.getParent(), CONFIG_DEFAULT);
      }

      if (fs.exists(configDefault)) {
        Configuration defaultConf = new XConfiguration(fs.open(configDefault));
        PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
        XConfiguration.injectDefaults(defaultConf, conf);
      } else {
        LOG.info("configDefault Doesn't exist " + configDefault);
      }
      PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);

      // Resolving all variables in the job properties.
      // This ensures the Hadoop Configuration semantics is preserved.
      XConfiguration resolvedVarsConf = new XConfiguration();
      for (Map.Entry<String, String> entry : conf) {
        resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
      }
      conf = resolvedVarsConf;
    } catch (IOException e) {
      throw new CommandException(
          ErrorCode.E0702,
          e.getMessage() + " : Problem reading default config " + configDefault,
          e);
    } catch (HadoopAccessorException e) {
      throw new CommandException(e);
    }
    LOG.debug("Merged CONF :" + XmlUtils.prettyPrint(conf).toString());
  }
コード例 #10
0
 private void setValue(String name, String value) {
   super.set(name, value);
 }
コード例 #11
0
  private LogChangesConfiguration loadConf() throws ServiceException {
    XConfiguration configuration;
    try {
      InputStream inputStream = getDefaultConfiguration();
      configuration = loadConfig(inputStream, true);
      File file = new File(configFile);
      if (!file.exists()) {
        log.info("Missing site configuration file [{0}]", configFile);
      } else {
        inputStream = new FileInputStream(configFile);
        XConfiguration siteConfiguration = loadConfig(inputStream, false);
        XConfiguration.injectDefaults(configuration, siteConfiguration);
        configuration = siteConfiguration;
      }
    } catch (IOException ex) {
      throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
    }

    if (log.isTraceEnabled()) {
      try {
        StringWriter writer = new StringWriter();
        for (Map.Entry<String, String> entry : configuration) {
          String value = getValue(configuration, entry.getKey());
          writer.write(" " + entry.getKey() + " = " + value + "\n");
        }
        writer.close();
        log.trace("Configuration:\n{0}---", writer.toString());
      } catch (IOException ex) {
        throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
      }
    }

    String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
    if (ignoreSysProps != null) {
      IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
    }

    for (Map.Entry<String, String> entry : configuration) {
      String sysValue = System.getProperty(entry.getKey());
      if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
        log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
        configuration.set(entry.getKey(), sysValue);
      }
    }
    for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
      String name = (String) entry.getKey();
      if (!IGNORE_SYS_PROPS.contains(name)) {
        if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
          if (configuration.get(name) == null) {
            log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
          }
        }
      }
    }

    // Backward compatible, we should still support -Dparam.
    for (String key : CONF_SYS_PROPS) {
      String sysValue = System.getProperty(key);
      if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) {
        log.info(
            "Overriding configuration with system property. Key [{0}], Value [{1}] ",
            key, sysValue);
        configuration.set(key, sysValue);
      }
    }

    return new LogChangesConfiguration(configuration);
  }
コード例 #12
0
  public void testGetGroupFromParent() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
    writer.write(APP1);
    writer.close();

    XConfiguration protoConf = getBaseProtoConf();
    final WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    String actionConf =
        "<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
            + "      <app-path>"
            + subWorkflowAppPath
            + File.separator
            + "workflow.xml"
            + "</app-path>"
            + "      <configuration>"
            + "        <property>"
            + "          <name>a</name>"
            + "          <value>A</value>"
            + "        </property>"
            + "      </configuration>"
            + "</sub-workflow>";

    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf(actionConf);

    // negative test
    final SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    workflow.setConf(newConf.toXmlString());

    subWorkflow.start(new Context(workflow, action), action);

    OozieClient oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(
        5000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            subWorkflow.check(new Context(workflow, action), action);
            return action.getStatus() == WorkflowActionBean.Status.DONE;
          }
        });

    subWorkflow.check(new Context(workflow, action), action);
    subWorkflow.end(new Context(workflow, action), action);

    assertEquals(WorkflowAction.Status.OK, action.getStatus());

    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));

    assertFalse(getTestGroup() == childConf.get(OozieClient.GROUP_NAME));

    // positive test
    newConf.set(OozieClient.GROUP_NAME, getTestGroup());
    workflow.setConf(newConf.toXmlString());
    final WorkflowActionBean action1 = new WorkflowActionBean();
    action1.setConf(actionConf);
    action1.setId("W1");

    subWorkflow.start(new Context(workflow, action1), action1);

    oozieClient =
        subWorkflow.getWorkflowClient(
            new Context(workflow, action1), SubWorkflowActionExecutor.LOCAL);

    waitFor(
        5000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            subWorkflow.check(new Context(workflow, action1), action1);
            return action1.getStatus() == WorkflowActionBean.Status.DONE;
          }
        });

    subWorkflow.check(new Context(workflow, action1), action1);
    subWorkflow.end(new Context(workflow, action1), action1);

    wf = oozieClient.getJobInfo(action1.getExternalId());
    childConf = new XConfiguration(new StringReader(wf.getConf()));
    assertEquals(getTestGroup(), childConf.get(OozieClient.GROUP_NAME));
  }