private void validateResourceRequest(ApplicationSubmissionContext submissionContext)
throws InvalidResourceRequestException {
// Validation of the ApplicationSubmissionContext needs to be completed
// here. Only those fields that are dependent on RM's configuration are
// checked here as they have to be validated whether they are part of new
// submission or just being recovered.
// Check whether AM resource requirements are within required limits
if (!submissionContext.getUnmanagedAM()) {
ResourceRequest amReq =
BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
ResourceRequest.ANY,
submissionContext.getResource(),
1);
try {
SchedulerUtils.validateResourceRequest(amReq, scheduler.getMaximumResourceCapability());
} catch (InvalidResourceRequestException e) {
LOG.warn(
"RM app submission failed in validating AM resource request"
+ " for application "
+ submissionContext.getApplicationId(),
e);
throw e;
}
}
}
private ResourceRequest validateAndCreateResourceRequest(
ApplicationSubmissionContext submissionContext, boolean isRecovery)
throws InvalidResourceRequestException {
// Validation of the ApplicationSubmissionContext needs to be completed
// here. Only those fields that are dependent on RM's configuration are
// checked here as they have to be validated whether they are part of new
// submission or just being recovered.
// Check whether AM resource requirements are within required limits
if (!submissionContext.getUnmanagedAM()) {
ResourceRequest amReq = submissionContext.getAMContainerResourceRequest();
if (amReq == null) {
amReq =
BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
ResourceRequest.ANY,
submissionContext.getResource(),
1);
}
// set label expression for AM container
if (null == amReq.getNodeLabelExpression()) {
amReq.setNodeLabelExpression(submissionContext.getNodeLabelExpression());
}
try {
SchedulerUtils.normalizeAndValidateRequest(
amReq,
scheduler.getMaximumResourceCapability(),
submissionContext.getQueue(),
scheduler,
isRecovery,
rmContext);
} catch (InvalidResourceRequestException e) {
LOG.warn(
"RM app submission failed in validating AM resource request"
+ " for application "
+ submissionContext.getApplicationId(),
e);
throw e;
}
SchedulerUtils.normalizeRequest(
amReq,
scheduler.getResourceCalculator(),
scheduler.getClusterResource(),
scheduler.getMinimumResourceCapability(),
scheduler.getMaximumResourceCapability(),
scheduler.getMinimumResourceCapability());
return amReq;
}
return null;
}
@Test(timeout = 500000)
public void testContainerUpdate() throws InterruptedException, IOException {
// Start the node
node.handle(
new RMNodeEvent(
null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
Configuration yarnconf = new YarnConfiguration();
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeAddedSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM)));
}
NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, "test", 0, 0, null, null, null);
node2.handle(
new RMNodeEvent(
null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeAddedSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
}
ContainerId completedContainerIdFromNode1 =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
ContainerId completedContainerIdFromNode2_1 =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1);
ContainerId completedContainerIdFromNode2_2 =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 2);
RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent();
RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent();
RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent();
ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);
doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1).getContainerId();
doReturn(Collections.singletonList(containerStatusFromNode1))
.when(statusEventFromNode1)
.getContainers();
doReturn(ContainerState.COMPLETE).when(containerStatusFromNode1).getState();
node.handle(statusEventFromNode1);
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeUpdateSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM)));
}
// ts.decCounter("test");
Assert.assertEquals(1, completedContainers.size());
Assert.assertEquals(completedContainerIdFromNode1, completedContainers.get(0).getContainerId());
completedContainers.clear();
doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1).getContainerId();
doReturn(Collections.singletonList(containerStatusFromNode2_1))
.when(statusEventFromNode2_1)
.getContainers();
doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_1).getState();
doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2).getContainerId();
doReturn(Collections.singletonList(containerStatusFromNode2_2))
.when(statusEventFromNode2_2)
.getContainers();
doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_2).getState();
node2.setNextHeartBeat(false);
node2.handle(statusEventFromNode2_1);
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
}
node2.setNextHeartBeat(true);
node2.handle(statusEventFromNode2_2);
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
}
// ts2.decCounter("test");
Assert.assertEquals(2, completedContainers.size());
Assert.assertEquals(
completedContainerIdFromNode2_1, completedContainers.get(0).getContainerId());
Assert.assertEquals(
completedContainerIdFromNode2_2, completedContainers.get(1).getContainerId());
}
