private void validateResourceRequest(ApplicationSubmissionContext submissionContext) throws InvalidResourceRequestException { // Validation of the ApplicationSubmissionContext needs to be completed // here. Only those fields that are dependent on RM's configuration are // checked here as they have to be validated whether they are part of new // submission or just being recovered. // Check whether AM resource requirements are within required limits if (!submissionContext.getUnmanagedAM()) { ResourceRequest amReq = BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1); try { SchedulerUtils.validateResourceRequest(amReq, scheduler.getMaximumResourceCapability()); } catch (InvalidResourceRequestException e) { LOG.warn( "RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e); throw e; } } }
private ResourceRequest validateAndCreateResourceRequest( ApplicationSubmissionContext submissionContext, boolean isRecovery) throws InvalidResourceRequestException { // Validation of the ApplicationSubmissionContext needs to be completed // here. Only those fields that are dependent on RM's configuration are // checked here as they have to be validated whether they are part of new // submission or just being recovered. // Check whether AM resource requirements are within required limits if (!submissionContext.getUnmanagedAM()) { ResourceRequest amReq = submissionContext.getAMContainerResourceRequest(); if (amReq == null) { amReq = BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1); } // set label expression for AM container if (null == amReq.getNodeLabelExpression()) { amReq.setNodeLabelExpression(submissionContext.getNodeLabelExpression()); } try { SchedulerUtils.normalizeAndValidateRequest( amReq, scheduler.getMaximumResourceCapability(), submissionContext.getQueue(), scheduler, isRecovery, rmContext); } catch (InvalidResourceRequestException e) { LOG.warn( "RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e); throw e; } SchedulerUtils.normalizeRequest( amReq, scheduler.getResourceCalculator(), scheduler.getClusterResource(), scheduler.getMinimumResourceCapability(), scheduler.getMaximumResourceCapability(), scheduler.getMinimumResourceCapability()); return amReq; } return null; }
@Test(timeout = 500000) public void testContainerUpdate() throws InterruptedException, IOException { // Start the node node.handle( new RMNodeEvent( null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM))); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler Configuration yarnconf = new YarnConfiguration(); if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeAddedSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM))); } NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1); RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, "test", 0, 0, null, null, null); node2.handle( new RMNodeEvent( null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM))); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeAddedSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM))); } ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0); ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1); ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 2); RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent(); RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent(); RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent(); ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class); ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class); ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class); doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1).getContainerId(); doReturn(Collections.singletonList(containerStatusFromNode1)) .when(statusEventFromNode1) .getContainers(); doReturn(ContainerState.COMPLETE).when(containerStatusFromNode1).getState(); node.handle(statusEventFromNode1); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeUpdateSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM))); } // ts.decCounter("test"); Assert.assertEquals(1, completedContainers.size()); Assert.assertEquals(completedContainerIdFromNode1, completedContainers.get(0).getContainerId()); completedContainers.clear(); doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1).getContainerId(); doReturn(Collections.singletonList(containerStatusFromNode2_1)) .when(statusEventFromNode2_1) .getContainers(); doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_1).getState(); doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2).getContainerId(); doReturn(Collections.singletonList(containerStatusFromNode2_2)) .when(statusEventFromNode2_2) .getContainers(); doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_2).getState(); node2.setNextHeartBeat(false); node2.handle(statusEventFromNode2_1); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM))); } node2.setNextHeartBeat(true); node2.handle(statusEventFromNode2_2); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM))); } // ts2.decCounter("test"); Assert.assertEquals(2, completedContainers.size()); Assert.assertEquals( completedContainerIdFromNode2_1, completedContainers.get(0).getContainerId()); Assert.assertEquals( completedContainerIdFromNode2_2, completedContainers.get(1).getContainerId()); }