@Test(timeout = 5000) public void testExpiredContainer() { // Start the node node.handle(new RMNodeEvent(null, RMNodeEventType.STARTED, null)); verify(scheduler).handle(any(NodeAddedSchedulerEvent.class)); // Expire a container ContainerId completedContainerId = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0); node.handle(new RMNodeCleanContainerEvent(null, completedContainerId, null)); Assert.assertEquals(1, node.getContainersToCleanUp().size()); // Now verify that scheduler isn't notified of an expired container // by checking number of 'completedContainers' it got in the previous event RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent(); ContainerStatus containerStatus = mock(ContainerStatus.class); doReturn(completedContainerId).when(containerStatus).getContainerId(); doReturn(Collections.singletonList(containerStatus)).when(statusEvent).getContainers(); node.handle(statusEvent); /* Expect the scheduler call handle function 2 times * 1. RMNode status from new to Running, handle the add_node event * 2. handle the node update event */ verify(scheduler, times(2)).handle(any(NodeUpdateSchedulerEvent.class)); }
@Test(timeout = 500000) public void testStatusChange() { // Start the node node.handle( new RMNodeEvent( null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM))); // Add info to the queue first node.setNextHeartBeat(false); ContainerId completedContainerId1 = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0); ContainerId completedContainerId2 = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1); RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent(); RMNodeStatusEvent statusEvent2 = getMockRMNodeStatusEvent(); ContainerStatus containerStatus1 = mock(ContainerStatus.class); ContainerStatus containerStatus2 = mock(ContainerStatus.class); doReturn(completedContainerId1).when(containerStatus1).getContainerId(); doReturn(Collections.singletonList(containerStatus1)).when(statusEvent1).getContainers(); doReturn(ContainerState.COMPLETE).when(containerStatus1).getState(); doReturn(completedContainerId2).when(containerStatus2).getContainerId(); doReturn(Collections.singletonList(containerStatus2)).when(statusEvent2).getContainers(); doReturn(ContainerState.COMPLETE).when(containerStatus2).getState(); verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class)); node.handle(statusEvent1); node.handle(statusEvent2); verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class)); Assert.assertEquals(2, node.getQueueSize()); node.handle( new RMNodeEvent( node.getNodeID(), RMNodeEventType.EXPIRE, new TransactionStateImpl(TransactionType.RM))); Assert.assertEquals(0, node.getQueueSize()); }
/** * Create and return a container object reflecting an allocation for the given appliction on the * given node with the given capability and priority. */ public Container createContainer(FSSchedulerNode node, Resource capability, Priority priority) { NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(getApplicationAttemptId(), getNewContainerId()); // Create the container Container container = BuilderUtils.newContainer( containerId, nodeId, node.getRMNode().getHttpAddress(), capability, priority, null); return container; }
public NodeHeartbeatResponse nodeHeartbeat( ApplicationAttemptId attemptId, long containerId, ContainerState containerState) throws Exception { HashMap<ApplicationId, List<ContainerStatus>> nodeUpdate = new HashMap<ApplicationId, List<ContainerStatus>>(1); ContainerStatus containerStatus = BuilderUtils.newContainerStatus( BuilderUtils.newContainerId(attemptId, containerId), containerState, "Success", 0); ArrayList<ContainerStatus> containerStatusList = new ArrayList<ContainerStatus>(1); containerStatusList.add(containerStatus); Log.info("ContainerStatus: " + containerStatus); nodeUpdate.put(attemptId.getApplicationId(), containerStatusList); return nodeHeartbeat(nodeUpdate, true); }
@Test(timeout = 2000000) public void testUpdateHeartbeatResponseForCleanup() throws IOException { RMNodeImpl node = getRunningNode(); NodeId nodeId = node.getNodeID(); int rpcID = HopYarnAPIUtilities.getRPCID(); byte[] allNMRequestData = new byte[1]; allNMRequestData[0] = 0xA; try { RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.RegisterNM, allNMRequestData); } catch (IOException ex) { Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex); } TransactionState ts = new TransactionStateImpl(TransactionType.RM); // Expire a container ContainerId completedContainerId = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0); node.handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId, ts)); ts.decCounter(TransactionState.TransactionType.INIT); Assert.assertEquals(1, node.getContainersToCleanUp().size()); rpcID = HopYarnAPIUtilities.getRPCID(); allNMRequestData = new byte[1]; allNMRequestData[0] = 0xA; try { RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.FinishApplicationMaster, allNMRequestData); } catch (IOException ex) { Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex); } TransactionState ts2 = new TransactionStateImpl(TransactionType.RM); // TransactionStateRM.newInstance(rpcID); // Finish an application ApplicationId finishedAppId = BuilderUtils.newApplicationId(0, 1); node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId, ts2)); ts2.decCounter(TransactionState.TransactionType.INIT); Assert.assertEquals(1, node.getAppsToCleanup().size()); rpcID = HopYarnAPIUtilities.getRPCID(); allNMRequestData = new byte[1]; allNMRequestData[0] = 0xA; try { RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.FinishApplicationMaster, allNMRequestData); } catch (IOException ex) { Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex); } TransactionState ts3 = new TransactionStateImpl(TransactionType.RM); // TransactionStateRM.newInstance(rpcID); // Verify status update does not clear containers/apps to cleanup // but updating heartbeat response for cleanup does RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent(); RMNodeStatusEvent se = new RMNodeStatusEvent( statusEvent.getNodeId(), statusEvent.getNodeHealthStatus(), statusEvent.getContainers(), statusEvent.getKeepAliveAppIds(), statusEvent.getLatestResponse(), ts3); node.handle(se); ts3.decCounter(TransactionState.TransactionType.INIT); Assert.assertEquals(1, node.getContainersToCleanUp().size()); Assert.assertEquals(1, node.getAppsToCleanup().size()); NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class); node.updateNodeHeartbeatResponseForCleanup(hbrsp, ts); Assert.assertEquals(0, node.getContainersToCleanUp().size()); Assert.assertEquals(0, node.getAppsToCleanup().size()); Assert.assertEquals(1, hbrsp.getContainersToCleanup().size()); Assert.assertEquals(completedContainerId, hbrsp.getContainersToCleanup().get(0)); Assert.assertEquals(1, hbrsp.getApplicationsToCleanup().size()); Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup().get(0)); }
@Test(timeout = 500000) public void testContainerUpdate() throws InterruptedException, IOException { // Start the node node.handle( new RMNodeEvent( null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM))); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler Configuration yarnconf = new YarnConfiguration(); if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeAddedSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM))); } NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1); RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, "test", 0, 0, null, null, null); node2.handle( new RMNodeEvent( null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM))); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeAddedSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM))); } ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0); ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1); ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId( BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 2); RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent(); RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent(); RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent(); ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class); ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class); ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class); doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1).getContainerId(); doReturn(Collections.singletonList(containerStatusFromNode1)) .when(statusEventFromNode1) .getContainers(); doReturn(ContainerState.COMPLETE).when(containerStatusFromNode1).getState(); node.handle(statusEventFromNode1); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeUpdateSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM))); } // ts.decCounter("test"); Assert.assertEquals(1, completedContainers.size()); Assert.assertEquals(completedContainerIdFromNode1, completedContainers.get(0).getContainerId()); completedContainers.clear(); doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1).getContainerId(); doReturn(Collections.singletonList(containerStatusFromNode2_1)) .when(statusEventFromNode2_1) .getContainers(); doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_1).getState(); doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2).getContainerId(); doReturn(Collections.singletonList(containerStatusFromNode2_2)) .when(statusEventFromNode2_2) .getContainers(); doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_2).getState(); node2.setNextHeartBeat(false); node2.handle(statusEventFromNode2_1); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM))); } node2.setNextHeartBeat(true); node2.handle(statusEventFromNode2_2); // If Distributed RT is enabled, this is the only way to let the scheduler // pick up the event, the PendingEvent retrieval does not invoke the // Mock Scheduler if (yarnconf.getBoolean( YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) { scheduler.handle( new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM))); } // ts2.decCounter("test"); Assert.assertEquals(2, completedContainers.size()); Assert.assertEquals( completedContainerIdFromNode2_1, completedContainers.get(0).getContainerId()); Assert.assertEquals( completedContainerIdFromNode2_2, completedContainers.get(1).getContainerId()); }