@Test(timeout = 5000)
public void testExpiredContainer() {
// Start the node
node.handle(new RMNodeEvent(null, RMNodeEventType.STARTED, null));
verify(scheduler).handle(any(NodeAddedSchedulerEvent.class));
// Expire a container
ContainerId completedContainerId =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
node.handle(new RMNodeCleanContainerEvent(null, completedContainerId, null));
Assert.assertEquals(1, node.getContainersToCleanUp().size());
// Now verify that scheduler isn't notified of an expired container
// by checking number of 'completedContainers' it got in the previous event
RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent();
ContainerStatus containerStatus = mock(ContainerStatus.class);
doReturn(completedContainerId).when(containerStatus).getContainerId();
doReturn(Collections.singletonList(containerStatus)).when(statusEvent).getContainers();
node.handle(statusEvent);
/* Expect the scheduler call handle function 2 times
* 1. RMNode status from new to Running, handle the add_node event
* 2. handle the node update event
*/
verify(scheduler, times(2)).handle(any(NodeUpdateSchedulerEvent.class));
}
@Test(timeout = 500000)
public void testStatusChange() {
// Start the node
node.handle(
new RMNodeEvent(
null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
// Add info to the queue first
node.setNextHeartBeat(false);
ContainerId completedContainerId1 =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
ContainerId completedContainerId2 =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1);
RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent();
RMNodeStatusEvent statusEvent2 = getMockRMNodeStatusEvent();
ContainerStatus containerStatus1 = mock(ContainerStatus.class);
ContainerStatus containerStatus2 = mock(ContainerStatus.class);
doReturn(completedContainerId1).when(containerStatus1).getContainerId();
doReturn(Collections.singletonList(containerStatus1)).when(statusEvent1).getContainers();
doReturn(ContainerState.COMPLETE).when(containerStatus1).getState();
doReturn(completedContainerId2).when(containerStatus2).getContainerId();
doReturn(Collections.singletonList(containerStatus2)).when(statusEvent2).getContainers();
doReturn(ContainerState.COMPLETE).when(containerStatus2).getState();
verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
node.handle(statusEvent1);
node.handle(statusEvent2);
verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
Assert.assertEquals(2, node.getQueueSize());
node.handle(
new RMNodeEvent(
node.getNodeID(),
RMNodeEventType.EXPIRE,
new TransactionStateImpl(TransactionType.RM)));
Assert.assertEquals(0, node.getQueueSize());
}
/**
* Create and return a container object reflecting an allocation for the given appliction on the
* given node with the given capability and priority.
*/
public Container createContainer(FSSchedulerNode node, Resource capability, Priority priority) {
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId =
BuilderUtils.newContainerId(getApplicationAttemptId(), getNewContainerId());
// Create the container
Container container =
BuilderUtils.newContainer(
containerId, nodeId, node.getRMNode().getHttpAddress(), capability, priority, null);
return container;
}
public NodeHeartbeatResponse nodeHeartbeat(
ApplicationAttemptId attemptId, long containerId, ContainerState containerState)
throws Exception {
HashMap<ApplicationId, List<ContainerStatus>> nodeUpdate =
new HashMap<ApplicationId, List<ContainerStatus>>(1);
ContainerStatus containerStatus =
BuilderUtils.newContainerStatus(
BuilderUtils.newContainerId(attemptId, containerId), containerState, "Success", 0);
ArrayList<ContainerStatus> containerStatusList = new ArrayList<ContainerStatus>(1);
containerStatusList.add(containerStatus);
Log.info("ContainerStatus: " + containerStatus);
nodeUpdate.put(attemptId.getApplicationId(), containerStatusList);
return nodeHeartbeat(nodeUpdate, true);
}
@Test(timeout = 2000000)
public void testUpdateHeartbeatResponseForCleanup() throws IOException {
RMNodeImpl node = getRunningNode();
NodeId nodeId = node.getNodeID();
int rpcID = HopYarnAPIUtilities.getRPCID();
byte[] allNMRequestData = new byte[1];
allNMRequestData[0] = 0xA;
try {
RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.RegisterNM, allNMRequestData);
} catch (IOException ex) {
Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex);
}
TransactionState ts = new TransactionStateImpl(TransactionType.RM);
// Expire a container
ContainerId completedContainerId =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
node.handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId, ts));
ts.decCounter(TransactionState.TransactionType.INIT);
Assert.assertEquals(1, node.getContainersToCleanUp().size());
rpcID = HopYarnAPIUtilities.getRPCID();
allNMRequestData = new byte[1];
allNMRequestData[0] = 0xA;
try {
RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.FinishApplicationMaster, allNMRequestData);
} catch (IOException ex) {
Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex);
}
TransactionState ts2 =
new TransactionStateImpl(TransactionType.RM); // TransactionStateRM.newInstance(rpcID);
// Finish an application
ApplicationId finishedAppId = BuilderUtils.newApplicationId(0, 1);
node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId, ts2));
ts2.decCounter(TransactionState.TransactionType.INIT);
Assert.assertEquals(1, node.getAppsToCleanup().size());
rpcID = HopYarnAPIUtilities.getRPCID();
allNMRequestData = new byte[1];
allNMRequestData[0] = 0xA;
try {
RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.FinishApplicationMaster, allNMRequestData);
} catch (IOException ex) {
Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex);
}
TransactionState ts3 =
new TransactionStateImpl(TransactionType.RM); // TransactionStateRM.newInstance(rpcID);
// Verify status update does not clear containers/apps to cleanup
// but updating heartbeat response for cleanup does
RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent();
RMNodeStatusEvent se =
new RMNodeStatusEvent(
statusEvent.getNodeId(),
statusEvent.getNodeHealthStatus(),
statusEvent.getContainers(),
statusEvent.getKeepAliveAppIds(),
statusEvent.getLatestResponse(),
ts3);
node.handle(se);
ts3.decCounter(TransactionState.TransactionType.INIT);
Assert.assertEquals(1, node.getContainersToCleanUp().size());
Assert.assertEquals(1, node.getAppsToCleanup().size());
NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class);
node.updateNodeHeartbeatResponseForCleanup(hbrsp, ts);
Assert.assertEquals(0, node.getContainersToCleanUp().size());
Assert.assertEquals(0, node.getAppsToCleanup().size());
Assert.assertEquals(1, hbrsp.getContainersToCleanup().size());
Assert.assertEquals(completedContainerId, hbrsp.getContainersToCleanup().get(0));
Assert.assertEquals(1, hbrsp.getApplicationsToCleanup().size());
Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup().get(0));
}
@Test(timeout = 500000)
public void testContainerUpdate() throws InterruptedException, IOException {
// Start the node
node.handle(
new RMNodeEvent(
null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
Configuration yarnconf = new YarnConfiguration();
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeAddedSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM)));
}
NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, "test", 0, 0, null, null, null);
node2.handle(
new RMNodeEvent(
null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeAddedSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
}
ContainerId completedContainerIdFromNode1 =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
ContainerId completedContainerIdFromNode2_1 =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1);
ContainerId completedContainerIdFromNode2_2 =
BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 2);
RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent();
RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent();
RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent();
ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);
doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1).getContainerId();
doReturn(Collections.singletonList(containerStatusFromNode1))
.when(statusEventFromNode1)
.getContainers();
doReturn(ContainerState.COMPLETE).when(containerStatusFromNode1).getState();
node.handle(statusEventFromNode1);
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeUpdateSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM)));
}
// ts.decCounter("test");
Assert.assertEquals(1, completedContainers.size());
Assert.assertEquals(completedContainerIdFromNode1, completedContainers.get(0).getContainerId());
completedContainers.clear();
doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1).getContainerId();
doReturn(Collections.singletonList(containerStatusFromNode2_1))
.when(statusEventFromNode2_1)
.getContainers();
doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_1).getState();
doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2).getContainerId();
doReturn(Collections.singletonList(containerStatusFromNode2_2))
.when(statusEventFromNode2_2)
.getContainers();
doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_2).getState();
node2.setNextHeartBeat(false);
node2.handle(statusEventFromNode2_1);
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
}
node2.setNextHeartBeat(true);
node2.handle(statusEventFromNode2_2);
// If Distributed RT is enabled, this is the only way to let the scheduler
// pick up the event, the PendingEvent retrieval does not invoke the
// Mock Scheduler
if (yarnconf.getBoolean(
YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
scheduler.handle(
new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
}
// ts2.decCounter("test");
Assert.assertEquals(2, completedContainers.size());
Assert.assertEquals(
completedContainerIdFromNode2_1, completedContainers.get(0).getContainerId());
Assert.assertEquals(
completedContainerIdFromNode2_2, completedContainers.get(1).getContainerId());
}
