void launchWorkerNode(Container container) throws IOException, YarnException { Map<String, String> env = System.getenv(); ContainerLaunchContext workerContext = Records.newRecord(ContainerLaunchContext.class); LocalResource workerJar = Records.newRecord(LocalResource.class); setupWorkerJar(workerJar); workerContext.setLocalResources(Collections.singletonMap("socialite.jar", workerJar)); System.out.println("[Master] workerJar:" + workerJar); Map<String, String> workerEnv = new HashMap<String, String>(env); setupWorkerEnv(workerEnv); workerContext.setEnvironment(workerEnv); String opts = "-Dsocialite.master=" + NetUtils.getHostname().split("/")[1] + " "; opts += "-Dsocialite.worker.num_threads=" + ClusterConf.get().getNumWorkerThreads() + " "; workerContext.setCommands( Collections.singletonList( "$JAVA_HOME/bin/java -ea " + env.get("JVM_OPTS") + " " + env.get("SOCIALITE_OPTS") + " " + opts + " " + "socialite.dist.worker.WorkerNode" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); nmClient.startContainer(container, workerContext); L.info("Launched worker node (container:" + container.getId() + ")"); }
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) throws Exception { YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName("test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(1); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); appContext.setUnmanagedAM(unmanaged); // Submit the application to the applications manager rmClient.submitApplication(appContext); return appId; }
public NodeHeartbeatResponse nodeHeartbeat( Map<ApplicationId, List<ContainerStatus>> conts, boolean isHealthy, int resId) throws Exception { NodeHeartbeatRequest req = Records.newRecord(NodeHeartbeatRequest.class); NodeStatus status = Records.newRecord(NodeStatus.class); status.setResponseId(resId); status.setNodeId(nodeId); for (Map.Entry<ApplicationId, List<ContainerStatus>> entry : conts.entrySet()) { Log.info("entry.getValue() " + entry.getValue()); status.setContainersStatuses(entry.getValue()); } NodeHealthStatus healthStatus = Records.newRecord(NodeHealthStatus.class); healthStatus.setHealthReport(""); healthStatus.setIsNodeHealthy(isHealthy); healthStatus.setLastHealthReportTime(1); status.setNodeHealthStatus(healthStatus); req.setNodeStatus(status); req.setLastKnownContainerTokenMasterKey(this.currentContainerTokenMasterKey); req.setLastKnownNMTokenMasterKey(this.currentNMTokenMasterKey); NodeHeartbeatResponse heartbeatResponse = resourceTracker.nodeHeartbeat(req); MasterKey masterKeyFromRM = heartbeatResponse.getContainerTokenMasterKey(); if (masterKeyFromRM != null && masterKeyFromRM.getKeyId() != this.currentContainerTokenMasterKey.getKeyId()) { this.currentContainerTokenMasterKey = masterKeyFromRM; } masterKeyFromRM = heartbeatResponse.getNMTokenMasterKey(); if (masterKeyFromRM != null && masterKeyFromRM.getKeyId() != this.currentNMTokenMasterKey.getKeyId()) { this.currentNMTokenMasterKey = masterKeyFromRM; } return heartbeatResponse; }
@Test public void testReportDiagnostics() throws Exception { JobID jobID = JobID.forName("job_1234567890000_0001"); JobId jobId = TypeConverter.toYarn(jobID); final String diagMsg = "some diagnostic message"; final JobDiagnosticsUpdateEvent diagUpdateEvent = new JobDiagnosticsUpdateEvent(jobId, diagMsg); MRAppMetrics mrAppMetrics = MRAppMetrics.create(); AppContext mockContext = mock(AppContext.class); when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true); JobImpl job = new JobImpl( jobId, Records.newRecord(ApplicationAttemptId.class), new Configuration(), mock(EventHandler.class), null, mock(JobTokenSecretManager.class), null, new SystemClock(), null, mrAppMetrics, null, true, null, 0, null, mockContext, null, null); job.handle(diagUpdateEvent); String diagnostics = job.getReport().getDiagnostics(); Assert.assertNotNull(diagnostics); Assert.assertTrue(diagnostics.contains(diagMsg)); job = new JobImpl( jobId, Records.newRecord(ApplicationAttemptId.class), new Configuration(), mock(EventHandler.class), null, mock(JobTokenSecretManager.class), null, new SystemClock(), null, mrAppMetrics, null, true, null, 0, null, mockContext, null, null); job.handle(new JobEvent(jobId, JobEventType.JOB_KILL)); job.handle(diagUpdateEvent); diagnostics = job.getReport().getDiagnostics(); Assert.assertNotNull(diagnostics); Assert.assertTrue(diagnostics.contains(diagMsg)); }
public void submitApplication( ApplicationId appId, String appName, Map<String, String> env, Map<String, LocalResource> localResources, List<String> commands, int memory) throws URISyntaxException, IOException { if (clientResourceManager == null) throw new IllegalStateException( "Cannot submit an application without connecting to resource manager!"); ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class); appCtx.setApplicationId(appId); appCtx.setApplicationName(appName); appCtx.setQueue("default"); appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); // System.out.println( "Based on my current user I am: " + // UserGroupInformation.getCurrentUser().getShortUserName() ); Priority prio = Records.newRecord(Priority.class); prio.setPriority(0); appCtx.setPriority(prio); /* for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) { System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource()); } */ /* for (Map.Entry<String, String> entry : env.entrySet()) { System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() ); } */ // Launch ctx ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class); containerCtx.setLocalResources(localResources); containerCtx.setCommands(commands); containerCtx.setEnvironment(env); containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); containerCtx.setResource(capability); appCtx.setAMContainerSpec(containerCtx); SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class); submitReq.setApplicationSubmissionContext(appCtx); LOG.info("Submitting application to ASM"); clientResourceManager.submitApplication(submitReq); // Don't return anything, ASM#submit returns an empty response }
public void launchSupervisorOnContainer(Container container) throws IOException { LOG.info("Connecting to ContainerManager for containerid=" + container.getId()); String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort(); InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr); LOG.info("Connecting to ContainerManager at " + cmIpPortStr); ContainerManager proxy = ((ContainerManager) rpc.getProxy(ContainerManager.class, cmAddress, hadoopConf)); LOG.info("launchSupervisorOnContainer( id:" + container.getId() + " )"); ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); launchContext.setContainerId(container.getId()); launchContext.setResource(container.getResource()); try { launchContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); } catch (IOException e) { LOG.info( "Getting current user info failed when trying to launch the container" + e.getMessage()); } Map<String, String> env = new HashMap<String, String>(); launchContext.setEnvironment(env); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String stormVersion = Util.getStormVersion(this.storm_conf); Path zip = new Path("/lib/storm/" + stormVersion + "/storm.zip"); FileSystem fs = FileSystem.get(this.hadoopConf); localResources.put( "storm", Util.newYarnAppResource( fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC)); String appHome = Util.getApplicationHomeForId(this.appAttemptId.toString()); Path dirDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf); localResources.put("conf", Util.newYarnAppResource(fs, dirDst)); launchContext.setLocalResources(localResources); List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf); launchContext.setCommands(supervisorArgs); StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(launchContext); LOG.info( "launchSupervisorOnContainer: startRequest prepared, calling startContainer. " + startRequest); try { StartContainerResponse response = proxy.startContainer(startRequest); LOG.info("Got a start container response " + response); } catch (Exception e) { LOG.error("Caught an exception while trying to start a container", e); System.exit(-1); } }
private GetJobReportResponse getJobReportResponse() { GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class); JobReport jobReport = Records.newRecord(JobReport.class); jobReport.setJobId(jobId); jobReport.setJobState(JobState.SUCCEEDED); jobReportResponse.setJobReport(jobReport); return jobReportResponse; }
@Test public void testNodeHeartBeatWithInvalidLabels() throws Exception { writeToHostsFile("host2"); Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath()); conf.set( YarnConfiguration.NODELABEL_CONFIGURATION_TYPE, YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE); final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager(); rm = new MockRM(conf) { @Override protected RMNodeLabelsManager createNodeLabelManager() { return nodeLabelsMgr; } }; rm.start(); try { nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); } ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService(); RegisterNodeManagerRequest registerReq = Records.newRecord(RegisterNodeManagerRequest.class); NodeId nodeId = NodeId.newInstance("host2", 1234); Resource capability = BuilderUtils.newResource(1024, 1); registerReq.setResource(capability); registerReq.setNodeId(nodeId); registerReq.setHttpPort(1234); registerReq.setNMVersion(YarnVersionInfo.getVersion()); registerReq.setNodeLabels(toNodeLabelSet("A")); RegisterNodeManagerResponse registerResponse = resourceTrackerService.registerNodeManager(registerReq); NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class); heartbeatReq.setNodeLabels(toNodeLabelSet("B", "#C")); // Invalid heart beat labels heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId)); heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey()); heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey()); NodeHeartbeatResponse nodeHeartbeatResponse = resourceTrackerService.nodeHeartbeat(heartbeatReq); // response should be NORMAL when RM heartbeat labels are rejected Assert.assertEquals( "Response should be NORMAL when RM heartbeat labels" + " are rejected", NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction()); Assert.assertFalse(nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM()); Assert.assertNotNull(nodeHeartbeatResponse.getDiagnosticsMessage()); rm.stop(); }
public void killApplication(String appId) throws Exception { KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class); ApplicationId aid = Records.newRecord(ApplicationId.class); long ts = Long.parseLong(appId.substring(appId.indexOf('_') + 1, appId.lastIndexOf('_'))); aid.setClusterTimestamp(ts); int id = Integer.parseInt(appId.substring(appId.lastIndexOf('_') + 1)); aid.setId(id); killRequest.setApplicationId(aid); crmp.forceKillApplication(killRequest); }
private GetJobReportResponse getJobReportResponseFromHistoryServer() { GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class); JobReport jobReport = Records.newRecord(JobReport.class); jobReport.setJobId(jobId); jobReport.setJobState(JobState.SUCCEEDED); jobReport.setMapProgress(1.0f); jobReport.setReduceProgress(1.0f); jobReport.setJobFile("TestJobFilePath"); jobReport.setTrackingUrl("http://TestTrackingUrl"); jobReportResponse.setJobReport(jobReport); return jobReportResponse; }
@Test public void testNodeHeartbeatWithCentralLabelConfig() throws Exception { writeToHostsFile("host2"); Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath()); conf.set( YarnConfiguration.NODELABEL_CONFIGURATION_TYPE, YarnConfiguration.DEFAULT_NODELABEL_CONFIGURATION_TYPE); final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager(); rm = new MockRM(conf) { @Override protected RMNodeLabelsManager createNodeLabelManager() { return nodeLabelsMgr; } }; rm.start(); ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService(); RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class); NodeId nodeId = NodeId.newInstance("host2", 1234); Resource capability = BuilderUtils.newResource(1024, 1); req.setResource(capability); req.setNodeId(nodeId); req.setHttpPort(1234); req.setNMVersion(YarnVersionInfo.getVersion()); req.setNodeLabels(toNodeLabelSet("A", "B", "C")); RegisterNodeManagerResponse registerResponse = resourceTrackerService.registerNodeManager(req); NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class); heartbeatReq.setNodeLabels(toNodeLabelSet("B")); // Valid heart beat labels heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId)); heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey()); heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey()); NodeHeartbeatResponse nodeHeartbeatResponse = resourceTrackerService.nodeHeartbeat(heartbeatReq); // response should be ok but the RMacceptNodeLabelsUpdate should be false Assert.assertEquals(NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction()); // no change in the labels, Assert.assertNull(nodeLabelsMgr.getNodeLabels().get(nodeId)); // heartbeat labels rejected Assert.assertFalse( "Invalid Node Labels should not accepted by RM", nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM()); if (rm != null) { rm.stop(); } }
/** * Setup a container request on specified node * * @param node the specified node * @return ResourceRequest sent to RM */ private ResourceRequest setupAContainerAskForRM(String node) { ResourceRequest request = Records.newRecord(ResourceRequest.class); request.setResourceName(node); request.setNumContainers(1); // important Priority priority = Records.newRecord(Priority.class); priority.setPriority(requestPriority); request.setPriority(priority); Resource capability = Records.newRecord(Resource.class); capability.setMemory(containerMemory); request.setCapability(capability); return request; }
/** * Utility method creating a {@link ResourceRequest}. * * @param numContainers number of containers to request * @return request to be sent to resource manager */ private ResourceRequest getContainerResourceRequest( int numContainers, String hostName, boolean relaxLocality) { ResourceRequest request = Records.newRecord(ResourceRequest.class); request.setRelaxLocality(relaxLocality); request.setResourceName(hostName); request.setNumContainers(numContainers); Priority pri = Records.newRecord(Priority.class); pri.setPriority(priority); request.setPriority(pri); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); ResourceCompat.setVirtualCores(capability, virtualcores); request.setCapability(capability); return request; }
private GetCountersResponse getCountersResponseFromHistoryServer() { GetCountersResponse countersResponse = Records.newRecord(GetCountersResponse.class); Counter counter = Records.newRecord(Counter.class); CounterGroup counterGroup = Records.newRecord(CounterGroup.class); Counters counters = Records.newRecord(Counters.class); counter.setDisplayName("dummyCounter"); counter.setName("dummyCounter"); counter.setValue(1001); counterGroup.setName("dummyCounters"); counterGroup.setDisplayName("dummyCounters"); counterGroup.setCounter("dummyCounter", counter); counters.setCounterGroup("dummyCounters", counterGroup); countersResponse.setCounters(counters); return countersResponse; }
public void runMainLoop() throws Exception { rmClient.registerApplicationMaster("", 0, ""); initRegistry(); startMasterNode(); Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); Resource capability = Records.newRecord(Resource.class); capability.setMemory(ClusterConf.get().getWorkerHeapSize()); capability.setVirtualCores(ClusterConf.get().getNumWorkerThreads()); List<ContainerRequest> containerReq = new ArrayList<ContainerRequest>(); for (int i = 0; i < ClusterConf.get().getNumWorkers(); ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); rmClient.addContainerRequest(containerAsk); containerReq.add(containerAsk); } while (alive) { Thread.sleep(1000); } finish(); /* for (ContainerRequest req : containerReq) { rmClient.removeContainerRequest(req); } int containersToAdd = 2; numContainersToWaitFor = containersToAdd; System.out.println("[Am] finished all containers. Asking for more containers, total=" + numContainersToWaitFor); for (int i = 0; i < containersToAdd; ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); System.out.println("[AM] Making res-req " + i); rmClient.addContainerRequest(containerAsk); } System.out.println("[AM] waiting for containers to finish once more!!!"); while (!doneWithContainers()) { Thread.sleep(100); } */ }
private ContainerLaunchContext newContainerLaunchContext( Container container, String helixInstanceName) throws IOException { Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId); Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME); Map<String, LocalResource> resourceMap = Maps.newHashMap(); addContainerLocalResources( new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap); addContainerLocalResources( new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap); addContainerLocalResources( new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap); if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) { addRemoteAppFiles( this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap); } ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(resourceMap); containerLaunchContext.setEnvironment( YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); containerLaunchContext.setCommands( Lists.newArrayList(buildContainerCommand(container, helixInstanceName))); if (UserGroupInformation.isSecurityEnabled()) { containerLaunchContext.setTokens(this.tokens.duplicate()); } return containerLaunchContext; }
@Override public void killApplication(ApplicationId applicationId) throws YarnException, IOException { KillApplicationRequest request = Records.newRecord(KillApplicationRequest.class); request.setApplicationId(applicationId); try { int pollCount = 0; long startTime = System.currentTimeMillis(); while (true) { KillApplicationResponse response = rmClient.forceKillApplication(request); if (response.getIsKillCompleted()) { LOG.info("Killed application " + applicationId); break; } long elapsedMillis = System.currentTimeMillis() - startTime; if (enforceAsyncAPITimeout() && elapsedMillis >= this.asyncApiPollTimeoutMillis) { throw new YarnException( "Timed out while waiting for application " + applicationId + " to be killed."); } if (++pollCount % 10 == 0) { LOG.info("Waiting for application " + applicationId + " to be killed."); } Thread.sleep(asyncApiPollIntervalMillis); } } catch (InterruptedException e) { LOG.error("Interrupted while waiting for application " + applicationId + " to be killed."); } }
public ChildExecutor(ChildExecutionContext context) { this.context = context; this.taskAttemptId = context.getTaskAttemptId(); this.taskType = context.getTaskType(); attemptReport = Records.newRecord(TaskAttemptReport.class); gcUpdater = new GcTimeUpdater(); }
@Public @Stable public static KillApplicationRequest newInstance(ApplicationId applicationId) { KillApplicationRequest request = Records.newRecord(KillApplicationRequest.class); request.setApplicationId(applicationId); return request; }
@Test public void testNodeRegistrationVersionLessThanRM() throws Exception { writeToHostsFile("host2"); Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath()); conf.set(YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, "EqualToRM"); rm = new MockRM(conf); rm.start(); String nmVersion = "1.9.9"; ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService(); RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class); NodeId nodeId = NodeId.newInstance("host2", 1234); Resource capability = BuilderUtils.newResource(1024, 1); req.setResource(capability); req.setNodeId(nodeId); req.setHttpPort(1234); req.setNMVersion(nmVersion); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN, response.getNodeAction()); Assert.assertTrue( "Diagnostic message did not contain: 'Disallowed NodeManager " + "Version " + nmVersion + ", is less than the minimum version'", response .getDiagnosticsMessage() .contains( "Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version ")); }
@Override public Token getRMDelegationToken(Text renewer) throws YarnException, IOException { /* get the token from RM */ GetDelegationTokenRequest rmDTRequest = Records.newRecord(GetDelegationTokenRequest.class); rmDTRequest.setRenewer(renewer.toString()); GetDelegationTokenResponse response = rmClient.getDelegationToken(rmDTRequest); return response.getRMDelegationToken(); }
@Override public YarnClientApplication createApplication() throws YarnException, IOException { ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); GetNewApplicationResponse newApp = getNewApplication(); ApplicationId appId = newApp.getApplicationId(); context.setApplicationId(appId); return new YarnClientApplication(newApp, context); }
@Public @Unstable public static ApplicationAttemptId newInstance(ApplicationId appId, int attemptId) { ApplicationAttemptId appAttemptId = Records.newRecord(ApplicationAttemptId.class); appAttemptId.setApplicationId(appId); appAttemptId.setAttemptId(attemptId); appAttemptId.build(); return appAttemptId; }
public List<NodeReport> getClusterNodes() throws YarnRemoteException { if (clientResourceManager == null) throw new IllegalArgumentException("Can't get report without connecting first!"); GetClusterNodesRequest req = Records.newRecord(GetClusterNodesRequest.class); GetClusterNodesResponse res = clientResourceManager.getClusterNodes(req); return res.getNodeReports(); }
/** * Changed the return type to AllocateResponse which use to hold a reference to AMResponse. * * <p>AMResponse seems to have disappeared in CDH 4.6 * * @param requestedContainers * @param releasedContainers * @return * @throws YarnRemoteException */ public AllocateResponse allocateRequest( List<ResourceRequest> requestedContainers, List<ContainerId> releasedContainers) throws YarnRemoteException { if (amResourceManager == null) throw new IllegalStateException( "Cannot send allocation request before connecting to the resource manager!"); LOG.info( "Sending allocation request" + ", requestedSize=" + requestedContainers.size() + ", releasedSize=" + releasedContainers.size()); for (ResourceRequest req : requestedContainers) LOG.info( "Requesting container, host=" + req.getHostName() + ", amount=" + req.getNumContainers() + ", memory=" + req.getCapability().getMemory() + ", priority=" + req.getPriority().getPriority()); for (ContainerId rel : releasedContainers) LOG.info("Releasing container: " + rel.getId()); AllocateRequest request = Records.newRecord(AllocateRequest.class); request.setResponseId(rmRequestId.incrementAndGet()); request.setApplicationAttemptId(appAttemptId); request.addAllAsks(requestedContainers); request.addAllReleases(releasedContainers); AllocateResponse response = amResourceManager.allocate(request); // response.getAllocatedContainers() LOG.debug( "Got an allocation response, " + ", responseId=" + response.getResponseId() + ", numClusterNodes=" + response.getNumClusterNodes() + ", headroom=" + response.getAvailableResources().getMemory() + ", allocatedSize=" + response.getAllocatedContainers().size() + ", updatedNodes=" + response.getUpdatedNodes().size() + ", reboot=" + response.getReboot() + ", completedSize=" + response.getCompletedContainersStatuses().size()); return response; }
@Public @Stable public static ResourceBlacklistRequest newInstance( List<String> additions, List<String> removals) { ResourceBlacklistRequest blacklistRequest = Records.newRecord(ResourceBlacklistRequest.class); blacklistRequest.setBlacklistAdditions(additions); blacklistRequest.setBlacklistRemovals(removals); return blacklistRequest; }
@Private @Unstable public static PreemptionContract newInstance( List<PreemptionResourceRequest> req, Set<PreemptionContainer> containers) { PreemptionContract contract = Records.newRecord(PreemptionContract.class); contract.setResourceRequest(req); contract.setContainers(containers); return contract; }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(requestPriority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(containerMemory); capability.setVirtualCores(containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: " + request.toString()); return request; }
@SuppressWarnings("unchecked") private NodeStatus getNodeStatusObject(NodeId nodeId) { NodeStatus status = Records.newRecord(NodeStatus.class); status.setNodeId(nodeId); status.setResponseId(0); status.setContainersStatuses(Collections.EMPTY_LIST); status.setKeepAliveApplications(Collections.EMPTY_LIST); return status; }
@Public @Unstable public static GetApplicationAttemptReportResponse newInstance( ApplicationAttemptReport ApplicationAttemptReport) { GetApplicationAttemptReportResponse response = Records.newRecord(GetApplicationAttemptReportResponse.class); response.setApplicationAttemptReport(ApplicationAttemptReport); return response; }