void launchWorkerNode(Container container) throws IOException, YarnException {
    Map<String, String> env = System.getenv();
    ContainerLaunchContext workerContext = Records.newRecord(ContainerLaunchContext.class);

    LocalResource workerJar = Records.newRecord(LocalResource.class);
    setupWorkerJar(workerJar);
    workerContext.setLocalResources(Collections.singletonMap("socialite.jar", workerJar));
    System.out.println("[Master] workerJar:" + workerJar);

    Map<String, String> workerEnv = new HashMap<String, String>(env);
    setupWorkerEnv(workerEnv);
    workerContext.setEnvironment(workerEnv);

    String opts = "-Dsocialite.master=" + NetUtils.getHostname().split("/")[1] + " ";
    opts += "-Dsocialite.worker.num_threads=" + ClusterConf.get().getNumWorkerThreads() + " ";
    workerContext.setCommands(
        Collections.singletonList(
            "$JAVA_HOME/bin/java -ea "
                + env.get("JVM_OPTS")
                + " "
                + env.get("SOCIALITE_OPTS")
                + " "
                + opts
                + " "
                + "socialite.dist.worker.WorkerNode"
                + " 1>"
                + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                + "/stdout"
                + " 2>"
                + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                + "/stderr"));
    nmClient.startContainer(container, workerContext);
    L.info("Launched worker node (container:" + container.getId() + ")");
  }
  private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) throws Exception {
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(1);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("default");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);

    return appId;
  }
  public NodeHeartbeatResponse nodeHeartbeat(
      Map<ApplicationId, List<ContainerStatus>> conts, boolean isHealthy, int resId)
      throws Exception {
    NodeHeartbeatRequest req = Records.newRecord(NodeHeartbeatRequest.class);
    NodeStatus status = Records.newRecord(NodeStatus.class);
    status.setResponseId(resId);
    status.setNodeId(nodeId);
    for (Map.Entry<ApplicationId, List<ContainerStatus>> entry : conts.entrySet()) {
      Log.info("entry.getValue() " + entry.getValue());
      status.setContainersStatuses(entry.getValue());
    }
    NodeHealthStatus healthStatus = Records.newRecord(NodeHealthStatus.class);
    healthStatus.setHealthReport("");
    healthStatus.setIsNodeHealthy(isHealthy);
    healthStatus.setLastHealthReportTime(1);
    status.setNodeHealthStatus(healthStatus);
    req.setNodeStatus(status);
    req.setLastKnownContainerTokenMasterKey(this.currentContainerTokenMasterKey);
    req.setLastKnownNMTokenMasterKey(this.currentNMTokenMasterKey);
    NodeHeartbeatResponse heartbeatResponse = resourceTracker.nodeHeartbeat(req);

    MasterKey masterKeyFromRM = heartbeatResponse.getContainerTokenMasterKey();
    if (masterKeyFromRM != null
        && masterKeyFromRM.getKeyId() != this.currentContainerTokenMasterKey.getKeyId()) {
      this.currentContainerTokenMasterKey = masterKeyFromRM;
    }

    masterKeyFromRM = heartbeatResponse.getNMTokenMasterKey();
    if (masterKeyFromRM != null
        && masterKeyFromRM.getKeyId() != this.currentNMTokenMasterKey.getKeyId()) {
      this.currentNMTokenMasterKey = masterKeyFromRM;
    }

    return heartbeatResponse;
  }
  @Test
  public void testReportDiagnostics() throws Exception {
    JobID jobID = JobID.forName("job_1234567890000_0001");
    JobId jobId = TypeConverter.toYarn(jobID);
    final String diagMsg = "some diagnostic message";
    final JobDiagnosticsUpdateEvent diagUpdateEvent = new JobDiagnosticsUpdateEvent(jobId, diagMsg);
    MRAppMetrics mrAppMetrics = MRAppMetrics.create();
    AppContext mockContext = mock(AppContext.class);
    when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
    JobImpl job =
        new JobImpl(
            jobId,
            Records.newRecord(ApplicationAttemptId.class),
            new Configuration(),
            mock(EventHandler.class),
            null,
            mock(JobTokenSecretManager.class),
            null,
            new SystemClock(),
            null,
            mrAppMetrics,
            null,
            true,
            null,
            0,
            null,
            mockContext,
            null,
            null);
    job.handle(diagUpdateEvent);
    String diagnostics = job.getReport().getDiagnostics();
    Assert.assertNotNull(diagnostics);
    Assert.assertTrue(diagnostics.contains(diagMsg));

    job =
        new JobImpl(
            jobId,
            Records.newRecord(ApplicationAttemptId.class),
            new Configuration(),
            mock(EventHandler.class),
            null,
            mock(JobTokenSecretManager.class),
            null,
            new SystemClock(),
            null,
            mrAppMetrics,
            null,
            true,
            null,
            0,
            null,
            mockContext,
            null,
            null);
    job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    job.handle(diagUpdateEvent);
    diagnostics = job.getReport().getDiagnostics();
    Assert.assertNotNull(diagnostics);
    Assert.assertTrue(diagnostics.contains(diagMsg));
  }
  public void submitApplication(
      ApplicationId appId,
      String appName,
      Map<String, String> env,
      Map<String, LocalResource> localResources,
      List<String> commands,
      int memory)
      throws URISyntaxException, IOException {

    if (clientResourceManager == null)
      throw new IllegalStateException(
          "Cannot submit an application without connecting to resource manager!");

    ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class);
    appCtx.setApplicationId(appId);
    appCtx.setApplicationName(appName);
    appCtx.setQueue("default");
    appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());

    // System.out.println( "Based on my current user I am: " +
    // UserGroupInformation.getCurrentUser().getShortUserName() );

    Priority prio = Records.newRecord(Priority.class);
    prio.setPriority(0);
    appCtx.setPriority(prio);
    /*
       for (Map.Entry<String, LocalResource> entry : localResources.entrySet())
       {
           System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource());
       }
    */
    /*
        for (Map.Entry<String, String> entry : env.entrySet())
        {
            System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() );
        }
    */

    // Launch ctx
    ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class);
    containerCtx.setLocalResources(localResources);
    containerCtx.setCommands(commands);
    containerCtx.setEnvironment(env);
    containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    containerCtx.setResource(capability);

    appCtx.setAMContainerSpec(containerCtx);

    SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class);
    submitReq.setApplicationSubmissionContext(appCtx);

    LOG.info("Submitting application to ASM");
    clientResourceManager.submitApplication(submitReq);

    // Don't return anything, ASM#submit returns an empty response
  }
Example #6
0
  public void launchSupervisorOnContainer(Container container) throws IOException {
    LOG.info("Connecting to ContainerManager for containerid=" + container.getId());
    String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
    InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
    LOG.info("Connecting to ContainerManager at " + cmIpPortStr);
    ContainerManager proxy =
        ((ContainerManager) rpc.getProxy(ContainerManager.class, cmAddress, hadoopConf));

    LOG.info("launchSupervisorOnContainer( id:" + container.getId() + " )");
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);

    launchContext.setContainerId(container.getId());
    launchContext.setResource(container.getResource());

    try {
      launchContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
    } catch (IOException e) {
      LOG.info(
          "Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    Map<String, String> env = new HashMap<String, String>();
    launchContext.setEnvironment(env);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    String stormVersion = Util.getStormVersion(this.storm_conf);
    Path zip = new Path("/lib/storm/" + stormVersion + "/storm.zip");
    FileSystem fs = FileSystem.get(this.hadoopConf);
    localResources.put(
        "storm",
        Util.newYarnAppResource(
            fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC));

    String appHome = Util.getApplicationHomeForId(this.appAttemptId.toString());

    Path dirDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf);

    localResources.put("conf", Util.newYarnAppResource(fs, dirDst));

    launchContext.setLocalResources(localResources);

    List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf);

    launchContext.setCommands(supervisorArgs);

    StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class);
    startRequest.setContainerLaunchContext(launchContext);

    LOG.info(
        "launchSupervisorOnContainer: startRequest prepared, calling startContainer. "
            + startRequest);
    try {
      StartContainerResponse response = proxy.startContainer(startRequest);
      LOG.info("Got a start container response " + response);
    } catch (Exception e) {
      LOG.error("Caught an exception while trying to start a container", e);
      System.exit(-1);
    }
  }
 private GetJobReportResponse getJobReportResponse() {
   GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class);
   JobReport jobReport = Records.newRecord(JobReport.class);
   jobReport.setJobId(jobId);
   jobReport.setJobState(JobState.SUCCEEDED);
   jobReportResponse.setJobReport(jobReport);
   return jobReportResponse;
 }
  @Test
  public void testNodeHeartBeatWithInvalidLabels() throws Exception {
    writeToHostsFile("host2");
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
    conf.set(
        YarnConfiguration.NODELABEL_CONFIGURATION_TYPE,
        YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);

    final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();

    rm =
        new MockRM(conf) {
          @Override
          protected RMNodeLabelsManager createNodeLabelManager() {
            return nodeLabelsMgr;
          }
        };
    rm.start();

    try {
      nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C"));
    } catch (IOException e) {
      Assert.fail("Caught Exception while intializing");
      e.printStackTrace();
    }

    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest registerReq = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = NodeId.newInstance("host2", 1234);
    Resource capability = BuilderUtils.newResource(1024, 1);
    registerReq.setResource(capability);
    registerReq.setNodeId(nodeId);
    registerReq.setHttpPort(1234);
    registerReq.setNMVersion(YarnVersionInfo.getVersion());
    registerReq.setNodeLabels(toNodeLabelSet("A"));
    RegisterNodeManagerResponse registerResponse =
        resourceTrackerService.registerNodeManager(registerReq);

    NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
    heartbeatReq.setNodeLabels(toNodeLabelSet("B", "#C")); // Invalid heart beat labels
    heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId));
    heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
    heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
    NodeHeartbeatResponse nodeHeartbeatResponse =
        resourceTrackerService.nodeHeartbeat(heartbeatReq);

    // response should be NORMAL when RM heartbeat labels are rejected
    Assert.assertEquals(
        "Response should be NORMAL when RM heartbeat labels" + " are rejected",
        NodeAction.NORMAL,
        nodeHeartbeatResponse.getNodeAction());
    Assert.assertFalse(nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
    Assert.assertNotNull(nodeHeartbeatResponse.getDiagnosticsMessage());
    rm.stop();
  }
 public void killApplication(String appId) throws Exception {
   KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
   ApplicationId aid = Records.newRecord(ApplicationId.class);
   long ts = Long.parseLong(appId.substring(appId.indexOf('_') + 1, appId.lastIndexOf('_')));
   aid.setClusterTimestamp(ts);
   int id = Integer.parseInt(appId.substring(appId.lastIndexOf('_') + 1));
   aid.setId(id);
   killRequest.setApplicationId(aid);
   crmp.forceKillApplication(killRequest);
 }
 private GetJobReportResponse getJobReportResponseFromHistoryServer() {
   GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class);
   JobReport jobReport = Records.newRecord(JobReport.class);
   jobReport.setJobId(jobId);
   jobReport.setJobState(JobState.SUCCEEDED);
   jobReport.setMapProgress(1.0f);
   jobReport.setReduceProgress(1.0f);
   jobReport.setJobFile("TestJobFilePath");
   jobReport.setTrackingUrl("http://TestTrackingUrl");
   jobReportResponse.setJobReport(jobReport);
   return jobReportResponse;
 }
  @Test
  public void testNodeHeartbeatWithCentralLabelConfig() throws Exception {
    writeToHostsFile("host2");
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
    conf.set(
        YarnConfiguration.NODELABEL_CONFIGURATION_TYPE,
        YarnConfiguration.DEFAULT_NODELABEL_CONFIGURATION_TYPE);

    final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();

    rm =
        new MockRM(conf) {
          @Override
          protected RMNodeLabelsManager createNodeLabelManager() {
            return nodeLabelsMgr;
          }
        };
    rm.start();

    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = NodeId.newInstance("host2", 1234);
    Resource capability = BuilderUtils.newResource(1024, 1);
    req.setResource(capability);
    req.setNodeId(nodeId);
    req.setHttpPort(1234);
    req.setNMVersion(YarnVersionInfo.getVersion());
    req.setNodeLabels(toNodeLabelSet("A", "B", "C"));
    RegisterNodeManagerResponse registerResponse = resourceTrackerService.registerNodeManager(req);

    NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
    heartbeatReq.setNodeLabels(toNodeLabelSet("B")); // Valid heart beat labels
    heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId));
    heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
    heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
    NodeHeartbeatResponse nodeHeartbeatResponse =
        resourceTrackerService.nodeHeartbeat(heartbeatReq);

    // response should be ok but the RMacceptNodeLabelsUpdate should be false
    Assert.assertEquals(NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction());
    // no change in the labels,
    Assert.assertNull(nodeLabelsMgr.getNodeLabels().get(nodeId));
    // heartbeat labels rejected
    Assert.assertFalse(
        "Invalid Node Labels should not accepted by RM",
        nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
    if (rm != null) {
      rm.stop();
    }
  }
  /**
   * Setup a container request on specified node
   *
   * @param node the specified node
   * @return ResourceRequest sent to RM
   */
  private ResourceRequest setupAContainerAskForRM(String node) {
    ResourceRequest request = Records.newRecord(ResourceRequest.class);
    request.setResourceName(node);
    request.setNumContainers(1); // important

    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(requestPriority);
    request.setPriority(priority);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(containerMemory);
    request.setCapability(capability);

    return request;
  }
 /**
  * Utility method creating a {@link ResourceRequest}.
  *
  * @param numContainers number of containers to request
  * @return request to be sent to resource manager
  */
 private ResourceRequest getContainerResourceRequest(
     int numContainers, String hostName, boolean relaxLocality) {
   ResourceRequest request = Records.newRecord(ResourceRequest.class);
   request.setRelaxLocality(relaxLocality);
   request.setResourceName(hostName);
   request.setNumContainers(numContainers);
   Priority pri = Records.newRecord(Priority.class);
   pri.setPriority(priority);
   request.setPriority(pri);
   Resource capability = Records.newRecord(Resource.class);
   capability.setMemory(memory);
   ResourceCompat.setVirtualCores(capability, virtualcores);
   request.setCapability(capability);
   return request;
 }
 private GetCountersResponse getCountersResponseFromHistoryServer() {
   GetCountersResponse countersResponse = Records.newRecord(GetCountersResponse.class);
   Counter counter = Records.newRecord(Counter.class);
   CounterGroup counterGroup = Records.newRecord(CounterGroup.class);
   Counters counters = Records.newRecord(Counters.class);
   counter.setDisplayName("dummyCounter");
   counter.setName("dummyCounter");
   counter.setValue(1001);
   counterGroup.setName("dummyCounters");
   counterGroup.setDisplayName("dummyCounters");
   counterGroup.setCounter("dummyCounter", counter);
   counters.setCounterGroup("dummyCounters", counterGroup);
   countersResponse.setCounters(counters);
   return countersResponse;
 }
  public void runMainLoop() throws Exception {

    rmClient.registerApplicationMaster("", 0, "");

    initRegistry();

    startMasterNode();

    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(ClusterConf.get().getWorkerHeapSize());
    capability.setVirtualCores(ClusterConf.get().getNumWorkerThreads());

    List<ContainerRequest> containerReq = new ArrayList<ContainerRequest>();
    for (int i = 0; i < ClusterConf.get().getNumWorkers(); ++i) {
      ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
      rmClient.addContainerRequest(containerAsk);
      containerReq.add(containerAsk);
    }

    while (alive) {
      Thread.sleep(1000);
    }
    finish();

    /*
    for (ContainerRequest req : containerReq) {
        rmClient.removeContainerRequest(req);
    }
    int containersToAdd = 2;
    numContainersToWaitFor = containersToAdd;

    System.out.println("[Am] finished all containers. Asking for more containers, total=" + numContainersToWaitFor);
    for (int i = 0; i < containersToAdd; ++i) {
        ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
        System.out.println("[AM] Making res-req " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    System.out.println("[AM] waiting for containers to finish once more!!!");
    while (!doneWithContainers()) {
        Thread.sleep(100);
    }
    */

  }
Example #16
0
  private ContainerLaunchContext newContainerLaunchContext(
      Container container, String helixInstanceName) throws IOException {
    Path appWorkDir =
        GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId);
    Path containerWorkDir =
        new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);

    Map<String, LocalResource> resourceMap = Maps.newHashMap();

    addContainerLocalResources(
        new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);
    addContainerLocalResources(
        new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap);
    addContainerLocalResources(
        new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap);

    if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) {
      addRemoteAppFiles(
          this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY),
          resourceMap);
    }

    ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
    containerLaunchContext.setLocalResources(resourceMap);
    containerLaunchContext.setEnvironment(
        YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
    containerLaunchContext.setCommands(
        Lists.newArrayList(buildContainerCommand(container, helixInstanceName)));

    if (UserGroupInformation.isSecurityEnabled()) {
      containerLaunchContext.setTokens(this.tokens.duplicate());
    }

    return containerLaunchContext;
  }
Example #17
0
  @Override
  public void killApplication(ApplicationId applicationId) throws YarnException, IOException {
    KillApplicationRequest request = Records.newRecord(KillApplicationRequest.class);
    request.setApplicationId(applicationId);

    try {
      int pollCount = 0;
      long startTime = System.currentTimeMillis();

      while (true) {
        KillApplicationResponse response = rmClient.forceKillApplication(request);
        if (response.getIsKillCompleted()) {
          LOG.info("Killed application " + applicationId);
          break;
        }

        long elapsedMillis = System.currentTimeMillis() - startTime;
        if (enforceAsyncAPITimeout() && elapsedMillis >= this.asyncApiPollTimeoutMillis) {
          throw new YarnException(
              "Timed out while waiting for application " + applicationId + " to be killed.");
        }

        if (++pollCount % 10 == 0) {
          LOG.info("Waiting for application " + applicationId + " to be killed.");
        }
        Thread.sleep(asyncApiPollIntervalMillis);
      }
    } catch (InterruptedException e) {
      LOG.error("Interrupted while waiting for application " + applicationId + " to be killed.");
    }
  }
Example #18
0
 public ChildExecutor(ChildExecutionContext context) {
   this.context = context;
   this.taskAttemptId = context.getTaskAttemptId();
   this.taskType = context.getTaskType();
   attemptReport = Records.newRecord(TaskAttemptReport.class);
   gcUpdater = new GcTimeUpdater();
 }
 @Public
 @Stable
 public static KillApplicationRequest newInstance(ApplicationId applicationId) {
   KillApplicationRequest request = Records.newRecord(KillApplicationRequest.class);
   request.setApplicationId(applicationId);
   return request;
 }
  @Test
  public void testNodeRegistrationVersionLessThanRM() throws Exception {
    writeToHostsFile("host2");
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
    conf.set(YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, "EqualToRM");
    rm = new MockRM(conf);
    rm.start();
    String nmVersion = "1.9.9";

    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = NodeId.newInstance("host2", 1234);
    Resource capability = BuilderUtils.newResource(1024, 1);
    req.setResource(capability);
    req.setNodeId(nodeId);
    req.setHttpPort(1234);
    req.setNMVersion(nmVersion);
    // trying to register a invalid node.
    RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
    Assert.assertEquals(NodeAction.SHUTDOWN, response.getNodeAction());
    Assert.assertTrue(
        "Diagnostic message did not contain: 'Disallowed NodeManager "
            + "Version "
            + nmVersion
            + ", is less than the minimum version'",
        response
            .getDiagnosticsMessage()
            .contains(
                "Disallowed NodeManager Version "
                    + nmVersion
                    + ", is less than the minimum version "));
  }
Example #21
0
 @Override
 public Token getRMDelegationToken(Text renewer) throws YarnException, IOException {
   /* get the token from RM */
   GetDelegationTokenRequest rmDTRequest = Records.newRecord(GetDelegationTokenRequest.class);
   rmDTRequest.setRenewer(renewer.toString());
   GetDelegationTokenResponse response = rmClient.getDelegationToken(rmDTRequest);
   return response.getRMDelegationToken();
 }
Example #22
0
 @Override
 public YarnClientApplication createApplication() throws YarnException, IOException {
   ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
   GetNewApplicationResponse newApp = getNewApplication();
   ApplicationId appId = newApp.getApplicationId();
   context.setApplicationId(appId);
   return new YarnClientApplication(newApp, context);
 }
 @Public
 @Unstable
 public static ApplicationAttemptId newInstance(ApplicationId appId, int attemptId) {
   ApplicationAttemptId appAttemptId = Records.newRecord(ApplicationAttemptId.class);
   appAttemptId.setApplicationId(appId);
   appAttemptId.setAttemptId(attemptId);
   appAttemptId.build();
   return appAttemptId;
 }
  public List<NodeReport> getClusterNodes() throws YarnRemoteException {
    if (clientResourceManager == null)
      throw new IllegalArgumentException("Can't get report without connecting first!");

    GetClusterNodesRequest req = Records.newRecord(GetClusterNodesRequest.class);
    GetClusterNodesResponse res = clientResourceManager.getClusterNodes(req);

    return res.getNodeReports();
  }
  /**
   * Changed the return type to AllocateResponse which use to hold a reference to AMResponse.
   *
   * <p>AMResponse seems to have disappeared in CDH 4.6
   *
   * @param requestedContainers
   * @param releasedContainers
   * @return
   * @throws YarnRemoteException
   */
  public AllocateResponse allocateRequest(
      List<ResourceRequest> requestedContainers, List<ContainerId> releasedContainers)
      throws YarnRemoteException {

    if (amResourceManager == null)
      throw new IllegalStateException(
          "Cannot send allocation request before connecting to the resource manager!");

    LOG.info(
        "Sending allocation request"
            + ", requestedSize="
            + requestedContainers.size()
            + ", releasedSize="
            + releasedContainers.size());

    for (ResourceRequest req : requestedContainers)
      LOG.info(
          "Requesting container, host="
              + req.getHostName()
              + ", amount="
              + req.getNumContainers()
              + ", memory="
              + req.getCapability().getMemory()
              + ", priority="
              + req.getPriority().getPriority());

    for (ContainerId rel : releasedContainers) LOG.info("Releasing container: " + rel.getId());

    AllocateRequest request = Records.newRecord(AllocateRequest.class);
    request.setResponseId(rmRequestId.incrementAndGet());
    request.setApplicationAttemptId(appAttemptId);
    request.addAllAsks(requestedContainers);
    request.addAllReleases(releasedContainers);

    AllocateResponse response = amResourceManager.allocate(request);

    // response.getAllocatedContainers()

    LOG.debug(
        "Got an allocation response, "
            + ", responseId="
            + response.getResponseId()
            + ", numClusterNodes="
            + response.getNumClusterNodes()
            + ", headroom="
            + response.getAvailableResources().getMemory()
            + ", allocatedSize="
            + response.getAllocatedContainers().size()
            + ", updatedNodes="
            + response.getUpdatedNodes().size()
            + ", reboot="
            + response.getReboot()
            + ", completedSize="
            + response.getCompletedContainersStatuses().size());

    return response;
  }
Example #26
0
 @Public
 @Stable
 public static ResourceBlacklistRequest newInstance(
     List<String> additions, List<String> removals) {
   ResourceBlacklistRequest blacklistRequest = Records.newRecord(ResourceBlacklistRequest.class);
   blacklistRequest.setBlacklistAdditions(additions);
   blacklistRequest.setBlacklistRemovals(removals);
   return blacklistRequest;
 }
Example #27
0
 @Private
 @Unstable
 public static PreemptionContract newInstance(
     List<PreemptionResourceRequest> req, Set<PreemptionContainer> containers) {
   PreemptionContract contract = Records.newRecord(PreemptionContract.class);
   contract.setResourceRequest(req);
   contract.setContainers(containers);
   return contract;
 }
  /**
   * Setup the request that will be sent to the RM for the container ask.
   *
   * @return the setup ResourceRequest to be sent to RM
   */
  private ContainerRequest setupContainerAskForRM() {
    // setup requirements for hosts
    // using * as any host will do for the distributed shell app
    // set the priority for the request
    Priority pri = Records.newRecord(Priority.class);
    // TODO - what is the range for priority? how to decide?
    pri.setPriority(requestPriority);

    // Set up resource type requirements
    // For now, memory and CPU are supported so we set memory and cpu requirements
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(containerMemory);
    capability.setVirtualCores(containerVirtualCores);

    ContainerRequest request = new ContainerRequest(capability, null, null, pri);
    LOG.info("Requested container ask: " + request.toString());
    return request;
  }
 @SuppressWarnings("unchecked")
 private NodeStatus getNodeStatusObject(NodeId nodeId) {
   NodeStatus status = Records.newRecord(NodeStatus.class);
   status.setNodeId(nodeId);
   status.setResponseId(0);
   status.setContainersStatuses(Collections.EMPTY_LIST);
   status.setKeepAliveApplications(Collections.EMPTY_LIST);
   return status;
 }
 @Public
 @Unstable
 public static GetApplicationAttemptReportResponse newInstance(
     ApplicationAttemptReport ApplicationAttemptReport) {
   GetApplicationAttemptReportResponse response =
       Records.newRecord(GetApplicationAttemptReportResponse.class);
   response.setApplicationAttemptReport(ApplicationAttemptReport);
   return response;
 }