コード例 #1
0
ファイル: Util.java プロジェクト: roshanp/storm-yarn
  @SuppressWarnings("rawtypes")
  static Path createConfigurationFileInFs(
      FileSystem fs, String appHome, Map stormConf, YarnConfiguration yarnConf) throws IOException {
    // dump stringwriter's content into FS conf/storm.yaml
    Path confDst =
        new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + STORM_CONF_PATH_STRING);
    Path dirDst = confDst.getParent();
    fs.mkdirs(dirDst);

    // storm.yaml
    FSDataOutputStream out = fs.create(confDst);
    Yaml yaml = new Yaml();
    OutputStreamWriter writer = new OutputStreamWriter(out);
    rmNulls(stormConf);
    yaml.dump(stormConf, writer);
    writer.close();
    out.close();

    // yarn-site.xml
    Path yarn_site_xml = new Path(dirDst, "yarn-site.xml");
    out = fs.create(yarn_site_xml);
    writer = new OutputStreamWriter(out);
    yarnConf.writeXml(writer);
    writer.close();
    out.close();

    // logback.xml
    Path logback_xml = new Path(dirDst, "logback.xml");
    out = fs.create(logback_xml);
    CreateLogbackXML(out);
    out.close();

    return dirDst;
  }
コード例 #2
0
ファイル: TestYARNRunner.java プロジェクト: wangxuemin/coding
  @Test(timeout = 20000)
  public void testWarnCommandOpts() throws Exception {
    Logger logger = Logger.getLogger(YARNRunner.class);

    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    Layout layout = new SimpleLayout();
    Appender appender = new WriterAppender(layout, bout);
    logger.addAppender(appender);

    JobConf jobConf = new JobConf();

    jobConf.set(
        MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
        "-Djava.net.preferIPv4Stack=true -Djava.library.path=foo");
    jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m -Djava.library.path=bar");

    YARNRunner yarnRunner = new YARNRunner(jobConf);

    File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
    OutputStream out = new FileOutputStream(jobxml);
    conf.writeXml(out);
    out.close();

    File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
    out = new FileOutputStream(jobsplit);
    out.close();

    File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO);
    out = new FileOutputStream(jobsplitmetainfo);
    out.close();

    File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE);
    out = new FileOutputStream(appTokens);
    out.close();

    @SuppressWarnings("unused")
    ApplicationSubmissionContext submissionContext =
        yarnRunner.createApplicationSubmissionContext(
            jobConf, testWorkDir.toString(), new Credentials());

    String logMsg = bout.toString();
    assertTrue(
        logMsg.contains(
            "WARN - Usage of -Djava.library.path in "
                + "yarn.app.mapreduce.am.admin-command-opts can cause programs to no "
                + "longer function if hadoop native libraries are used. These values "
                + "should be set as part of the LD_LIBRARY_PATH in the app master JVM "
                + "env using yarn.app.mapreduce.am.admin.user.env config settings."));
    assertTrue(
        logMsg.contains(
            "WARN - Usage of -Djava.library.path in "
                + "yarn.app.mapreduce.am.command-opts can cause programs to no longer "
                + "function if hadoop native libraries are used. These values should "
                + "be set as part of the LD_LIBRARY_PATH in the app master JVM env "
                + "using yarn.app.mapreduce.am.env config settings."));
  }
コード例 #3
0
ファイル: TestYARNRunner.java プロジェクト: wangxuemin/coding
 @Test(timeout = 20000)
 public void testJobSubmissionFailure() throws Exception {
   when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class)))
       .thenReturn(appId);
   ApplicationReport report = mock(ApplicationReport.class);
   when(report.getApplicationId()).thenReturn(appId);
   when(report.getDiagnostics()).thenReturn(failString);
   when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
   when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
   Credentials credentials = new Credentials();
   File jobxml = new File(testWorkDir, "job.xml");
   OutputStream out = new FileOutputStream(jobxml);
   conf.writeXml(out);
   out.close();
   try {
     yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
   } catch (IOException io) {
     LOG.info("Logging exception:", io);
     assertTrue(io.getLocalizedMessage().contains(failString));
   }
 }
コード例 #4
0
ファイル: TestYARNRunner.java プロジェクト: wangxuemin/coding
  @Test(timeout = 20000)
  public void testAMAdminCommandOpts() throws Exception {
    JobConf jobConf = new JobConf();

    jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true");
    jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m");

    YARNRunner yarnRunner = new YARNRunner(jobConf);

    File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
    OutputStream out = new FileOutputStream(jobxml);
    conf.writeXml(out);
    out.close();

    File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
    out = new FileOutputStream(jobsplit);
    out.close();

    File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO);
    out = new FileOutputStream(jobsplitmetainfo);
    out.close();

    File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE);
    out = new FileOutputStream(appTokens);
    out.close();

    ApplicationSubmissionContext submissionContext =
        yarnRunner.createApplicationSubmissionContext(
            jobConf, testWorkDir.toString(), new Credentials());

    ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
    List<String> commands = containerSpec.getCommands();

    int index = 0;
    int adminIndex = 0;
    int adminPos = -1;
    int userIndex = 0;
    int userPos = -1;

    for (String command : commands) {
      if (command != null) {
        adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true");
        if (adminPos >= 0) adminIndex = index;

        userPos = command.indexOf("-Xmx1024m");
        if (userPos >= 0) userIndex = index;
      }

      index++;
    }

    // Check both admin java opts and user java opts are in the commands
    assertTrue("AM admin command opts not in the commands.", adminPos > 0);
    assertTrue("AM user command opts not in the commands.", userPos > 0);

    // Check the admin java opts is before user java opts in the commands
    if (adminIndex == userIndex) {
      assertTrue("AM admin command opts is after user command opts.", adminPos < userPos);
    } else {
      assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex);
    }
  }