public void submitApplication(
      ApplicationId appId,
      String appName,
      Map<String, String> env,
      Map<String, LocalResource> localResources,
      List<String> commands,
      int memory)
      throws URISyntaxException, IOException {

    if (clientResourceManager == null)
      throw new IllegalStateException(
          "Cannot submit an application without connecting to resource manager!");

    ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class);
    appCtx.setApplicationId(appId);
    appCtx.setApplicationName(appName);
    appCtx.setQueue("default");
    appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());

    // System.out.println( "Based on my current user I am: " +
    // UserGroupInformation.getCurrentUser().getShortUserName() );

    Priority prio = Records.newRecord(Priority.class);
    prio.setPriority(0);
    appCtx.setPriority(prio);
    /*
       for (Map.Entry<String, LocalResource> entry : localResources.entrySet())
       {
           System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource());
       }
    */
    /*
        for (Map.Entry<String, String> entry : env.entrySet())
        {
            System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() );
        }
    */

    // Launch ctx
    ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class);
    containerCtx.setLocalResources(localResources);
    containerCtx.setCommands(commands);
    containerCtx.setEnvironment(env);
    containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    containerCtx.setResource(capability);

    appCtx.setAMContainerSpec(containerCtx);

    SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class);
    submitReq.setApplicationSubmissionContext(appCtx);

    LOG.info("Submitting application to ASM");
    clientResourceManager.submitApplication(submitReq);

    // Don't return anything, ASM#submit returns an empty response
  }
Esempio n. 2
0
  public void launchSupervisorOnContainer(Container container) throws IOException {
    LOG.info("Connecting to ContainerManager for containerid=" + container.getId());
    String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
    InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
    LOG.info("Connecting to ContainerManager at " + cmIpPortStr);
    ContainerManager proxy =
        ((ContainerManager) rpc.getProxy(ContainerManager.class, cmAddress, hadoopConf));

    LOG.info("launchSupervisorOnContainer( id:" + container.getId() + " )");
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);

    launchContext.setContainerId(container.getId());
    launchContext.setResource(container.getResource());

    try {
      launchContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
    } catch (IOException e) {
      LOG.info(
          "Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    Map<String, String> env = new HashMap<String, String>();
    launchContext.setEnvironment(env);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    String stormVersion = Util.getStormVersion(this.storm_conf);
    Path zip = new Path("/lib/storm/" + stormVersion + "/storm.zip");
    FileSystem fs = FileSystem.get(this.hadoopConf);
    localResources.put(
        "storm",
        Util.newYarnAppResource(
            fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC));

    String appHome = Util.getApplicationHomeForId(this.appAttemptId.toString());

    Path dirDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf);

    localResources.put("conf", Util.newYarnAppResource(fs, dirDst));

    launchContext.setLocalResources(localResources);

    List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf);

    launchContext.setCommands(supervisorArgs);

    StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class);
    startRequest.setContainerLaunchContext(launchContext);

    LOG.info(
        "launchSupervisorOnContainer: startRequest prepared, calling startContainer. "
            + startRequest);
    try {
      StartContainerResponse response = proxy.startContainer(startRequest);
      LOG.info("Got a start container response " + response);
    } catch (Exception e) {
      LOG.error("Caught an exception while trying to start a container", e);
      System.exit(-1);
    }
  }
Esempio n. 3
0
  @Test
  public void testSuccessfulContainerLaunch() throws InterruptedException, IOException {

    FileContext localFS = FileContext.getLocalFSFileContext();

    localFS.delete(new Path(localDir.getAbsolutePath()), true);
    localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
    localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
    localDir.mkdir();
    localLogDir.mkdir();
    remoteLogDir.mkdir();

    Context context = new NMContext();

    YarnConfiguration conf = new YarnConfiguration();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());

    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.setConf(conf);

    DeletionService del = new DeletionService(exec);
    Dispatcher dispatcher = new AsyncDispatcher();
    NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
    NodeManagerMetrics metrics = NodeManagerMetrics.create();
    ContainerTokenSecretManager containerTokenSecretManager = new ContainerTokenSecretManager();
    NodeStatusUpdater nodeStatusUpdater =
        new NodeStatusUpdaterImpl(
            context, dispatcher, healthChecker, metrics, containerTokenSecretManager) {
          @Override
          protected ResourceTracker getRMClient() {
            return new LocalRMInterface();
          };

          @Override
          protected void startStatusUpdater() {
            return; // Don't start any updating thread.
          }
        };

    DummyContainerManager containerManager =
        new DummyContainerManager(
            context,
            exec,
            del,
            nodeStatusUpdater,
            metrics,
            containerTokenSecretManager,
            new ApplicationACLsManager(conf),
            dirsHandler);
    containerManager.init(conf);
    containerManager.start();

    ContainerLaunchContext launchContext =
        recordFactory.newRecordInstance(ContainerLaunchContext.class);
    ContainerId cID = recordFactory.newRecordInstance(ContainerId.class);
    ApplicationId applicationId = recordFactory.newRecordInstance(ApplicationId.class);
    applicationId.setClusterTimestamp(0);
    applicationId.setId(0);
    ApplicationAttemptId applicationAttemptId =
        recordFactory.newRecordInstance(ApplicationAttemptId.class);
    applicationAttemptId.setApplicationId(applicationId);
    applicationAttemptId.setAttemptId(0);
    cID.setApplicationAttemptId(applicationAttemptId);
    launchContext.setContainerId(cID);
    launchContext.setUser("testing");
    launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
    StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
    request.setContainerLaunchContext(launchContext);
    containerManager.startContainer(request);

    BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.RUNNING);

    StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
    stopRequest.setContainerId(cID);
    containerManager.stopContainer(stopRequest);
    BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE);

    containerManager.stop();
  }