/**
   * Basic idea of the test: 1. register a token for 2 seconds with no cancel at the end 2. cancel
   * it immediately 3. Sleep and check that the 2 seconds renew didn't happen (totally 5 renewals)
   * 4. check cancellation
   *
   * @throws IOException
   * @throws URISyntaxException
   */
  @Test(timeout = 60000)
  public void testDTRenewalWithNoCancel() throws Exception {
    MyFS dfs = (MyFS) FileSystem.get(conf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());

    Credentials ts = new Credentials();
    MyToken token1 = dfs.getDelegationToken(new Text("user1"));

    // to cause this one to be set for renew in 2 secs
    Renewer.tokenToRenewIn2Sec = token1;
    LOG.info("token=" + token1 + " should be renewed for 2 secs");

    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    ts.addToken(new Text(nn1), token1);

    ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1);
    delegationTokenRenewer.addApplication(applicationId_1, ts, false, false);
    waitForEventsToGetProcessed(delegationTokenRenewer);
    delegationTokenRenewer.applicationFinished(applicationId_1);
    waitForEventsToGetProcessed(delegationTokenRenewer);
    int numberOfExpectedRenewals = Renewer.counter; // number of renewals so far
    try {
      Thread.sleep(6 * 1000); // sleep 6 seconds, so it has time to renew
    } catch (InterruptedException e) {
    }
    LOG.info("Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);

    // counter and the token should still be the old ones
    assertEquals(
        "renew wasn't called as many times as expected", numberOfExpectedRenewals, Renewer.counter);

    // also renewing of the canceled token should not fail, because it has not
    // been canceled
    token1.renew(conf);
  }
  @Test(timeout = 5000)
  public void testExpiredContainer() {
    // Start the node
    node.handle(new RMNodeEvent(null, RMNodeEventType.STARTED, null));
    verify(scheduler).handle(any(NodeAddedSchedulerEvent.class));

    // Expire a container
    ContainerId completedContainerId =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    node.handle(new RMNodeCleanContainerEvent(null, completedContainerId, null));
    Assert.assertEquals(1, node.getContainersToCleanUp().size());

    // Now verify that scheduler isn't notified of an expired container
    // by checking number of 'completedContainers' it got in the previous event
    RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent();
    ContainerStatus containerStatus = mock(ContainerStatus.class);
    doReturn(completedContainerId).when(containerStatus).getContainerId();
    doReturn(Collections.singletonList(containerStatus)).when(statusEvent).getContainers();
    node.handle(statusEvent);
    /* Expect the scheduler call handle function 2 times
     * 1. RMNode status from new to Running, handle the add_node event
     * 2. handle the node update event
     */
    verify(scheduler, times(2)).handle(any(NodeUpdateSchedulerEvent.class));
  }
  @Test(timeout = 60000)
  public void testAppRejectionWithCancelledDelegationToken() throws Exception {
    MyFS dfs = (MyFS) FileSystem.get(conf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());

    MyToken token = dfs.getDelegationToken(new Text("user1"));
    token.cancelToken();

    Credentials ts = new Credentials();
    ts.addToken(token.getKind(), token);

    // register the tokens for renewal
    ApplicationId appId = BuilderUtils.newApplicationId(0, 0);
    delegationTokenRenewer.addApplication(appId, ts, true, false);
    int waitCnt = 20;
    while (waitCnt-- > 0) {
      if (!eventQueue.isEmpty()) {
        Event evt = eventQueue.take();
        if (evt.getType() == RMAppEventType.APP_REJECTED) {
          Assert.assertTrue(((RMAppEvent) evt).getApplicationId().equals(appId));
          return;
        }
      } else {
        Thread.sleep(500);
      }
    }
    fail("App submission with a cancelled token should have failed");
  }
  /**
   * Basic idea of the test: 0. Setup token KEEP_ALIVE 1. create tokens. 2. register them for
   * renewal - to be cancelled on app complete 3. Complete app. 4. Verify token is alive within the
   * KEEP_ALIVE time 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
   *
   * @throws IOException
   * @throws URISyntaxException
   */
  @Test(timeout = 60000)
  public void testDTKeepAlive1() throws Exception {
    Configuration lconf = new Configuration(conf);
    lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    // Keep tokens alive for 6 seconds.
    lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
    // Try removing tokens every second.
    lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l);
    DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(lconf, counter);
    localDtr.init(lconf);
    RMContext mockContext = mock(RMContext.class);
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr);
    when(mockContext.getDispatcher()).thenReturn(dispatcher);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    localDtr.setRMContext(mockContext);
    localDtr.start();

    MyFS dfs = (MyFS) FileSystem.get(lconf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode());

    Credentials ts = new Credentials();
    // get the delegation tokens
    MyToken token1 = dfs.getDelegationToken(new Text("user1"));

    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    ts.addToken(new Text(nn1), token1);

    // register the tokens for renewal
    ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
    localDtr.addApplication(applicationId_0, ts, true, false);
    waitForEventsToGetProcessed(localDtr);
    if (!eventQueue.isEmpty()) {
      Event evt = eventQueue.take();
      if (evt instanceof RMAppEvent) {
        Assert.assertEquals(((RMAppEvent) evt).getType(), RMAppEventType.START);
      } else {
        fail("RMAppEvent.START was expected!!");
      }
    }

    localDtr.applicationFinished(applicationId_0);
    waitForEventsToGetProcessed(localDtr);

    // Token should still be around. Renewal should not fail.
    token1.renew(lconf);

    // Allow the keepalive time to run out
    Thread.sleep(10000l);

    // The token should have been cancelled at this point. Renewal will fail.
    try {
      token1.renew(lconf);
      fail("Renewal of cancelled token should have failed");
    } catch (InvalidToken ite) {
    }
  }
  @Test(timeout = 500000)
  public void testStatusChange() {
    // Start the node
    node.handle(
        new RMNodeEvent(
            null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
    // Add info to the queue first
    node.setNextHeartBeat(false);

    ContainerId completedContainerId1 =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    ContainerId completedContainerId2 =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1);

    RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent();
    RMNodeStatusEvent statusEvent2 = getMockRMNodeStatusEvent();

    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
    ContainerStatus containerStatus2 = mock(ContainerStatus.class);

    doReturn(completedContainerId1).when(containerStatus1).getContainerId();
    doReturn(Collections.singletonList(containerStatus1)).when(statusEvent1).getContainers();
    doReturn(ContainerState.COMPLETE).when(containerStatus1).getState();
    doReturn(completedContainerId2).when(containerStatus2).getContainerId();
    doReturn(Collections.singletonList(containerStatus2)).when(statusEvent2).getContainers();
    doReturn(ContainerState.COMPLETE).when(containerStatus2).getState();
    verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
    node.handle(statusEvent1);
    node.handle(statusEvent2);
    verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
    Assert.assertEquals(2, node.getQueueSize());
    node.handle(
        new RMNodeEvent(
            node.getNodeID(),
            RMNodeEventType.EXPIRE,
            new TransactionStateImpl(TransactionType.RM)));
    Assert.assertEquals(0, node.getQueueSize());
  }
  /**
   * Basic idea of the test: 1. create tokens. 2. Mark one of them to be renewed in 2 seconds
   * (instead of 24 hours) 3. register them for renewal 4. sleep for 3 seconds 5. count number of
   * renewals (should 3 initial ones + one extra) 6. register another token for 2 seconds 7. cancel
   * it immediately 8. Sleep and check that the 2 seconds renew didn't happen (totally 5 renewals)
   * 9. check cancellation
   *
   * @throws IOException
   * @throws URISyntaxException
   */
  @Test(timeout = 60000)
  public void testDTRenewal() throws Exception {
    MyFS dfs = (MyFS) FileSystem.get(conf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
    // Test 1. - add three tokens - make sure exactly one get's renewed

    // get the delegation tokens
    MyToken token1, token2, token3;
    token1 = dfs.getDelegationToken(new Text("user1"));
    token2 = dfs.getDelegationToken(new Text("user2"));
    token3 = dfs.getDelegationToken(new Text("user3"));

    // to cause this one to be set for renew in 2 secs
    Renewer.tokenToRenewIn2Sec = token1;
    LOG.info("token=" + token1 + " should be renewed for 2 secs");

    // three distinct Namenodes
    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    String nn2 = DelegationTokenRenewer.SCHEME + "://host2:0";
    String nn3 = DelegationTokenRenewer.SCHEME + "://host3:0";

    Credentials ts = new Credentials();

    // register the token for renewal
    ts.addToken(new Text(nn1), token1);
    ts.addToken(new Text(nn2), token2);
    ts.addToken(new Text(nn3), token3);

    // register the tokens for renewal
    ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
    delegationTokenRenewer.addApplication(applicationId_0, ts, true, false);
    waitForEventsToGetProcessed(delegationTokenRenewer);

    // first 3 initial renewals + 1 real
    int numberOfExpectedRenewals = 3 + 1;

    int attempts = 10;
    while (attempts-- > 0) {
      try {
        Thread.sleep(3 * 1000); // sleep 3 seconds, so it has time to renew
      } catch (InterruptedException e) {
      }

      // since we cannot guarantee timely execution - let's give few chances
      if (Renewer.counter == numberOfExpectedRenewals) break;
    }

    LOG.info(
        "dfs=" + dfs.hashCode() + ";Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
    assertEquals(
        "renew wasn't called as many times as expected(4):",
        numberOfExpectedRenewals,
        Renewer.counter);
    assertEquals("most recently renewed token mismatch", Renewer.lastRenewed, token1);

    // Test 2.
    // add another token ( that expires in 2 secs). Then remove it, before
    // time is up.
    // Wait for 3 secs , and make sure no renews were called
    ts = new Credentials();
    MyToken token4 = dfs.getDelegationToken(new Text("user4"));

    // to cause this one to be set for renew in 2 secs
    Renewer.tokenToRenewIn2Sec = token4;
    LOG.info("token=" + token4 + " should be renewed for 2 secs");

    String nn4 = DelegationTokenRenewer.SCHEME + "://host4:0";
    ts.addToken(new Text(nn4), token4);

    ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1);
    delegationTokenRenewer.addApplication(applicationId_1, ts, true, false);
    waitForEventsToGetProcessed(delegationTokenRenewer);
    delegationTokenRenewer.applicationFinished(applicationId_1);
    waitForEventsToGetProcessed(delegationTokenRenewer);
    numberOfExpectedRenewals = Renewer.counter; // number of renewals so far
    try {
      Thread.sleep(6 * 1000); // sleep 6 seconds, so it has time to renew
    } catch (InterruptedException e) {
    }
    LOG.info("Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);

    // counter and the token should stil be the old ones
    assertEquals(
        "renew wasn't called as many times as expected", numberOfExpectedRenewals, Renewer.counter);

    // also renewing of the cancelled token should fail
    try {
      token4.renew(conf);
      fail("Renewal of cancelled token should have failed");
    } catch (InvalidToken ite) {
      // expected
    }
  }
  @Test
  public void testApplicationOrderingWithPriority() throws Exception {

    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

    LeafQueue q = (LeafQueue) cs.getQueue("default");
    Assert.assertNotNull(q);

    String host = "127.0.0.1";
    RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(16 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));

    // add app 1 start
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1);

    RMAppAttemptMetrics attemptMetric1 = new RMAppAttemptMetrics(appAttemptId1, rm.getRMContext());
    RMAppImpl app1 = mock(RMAppImpl.class);
    when(app1.getApplicationId()).thenReturn(appId1);
    RMAppAttemptImpl attempt1 = mock(RMAppAttemptImpl.class);
    when(attempt1.getAppAttemptId()).thenReturn(appAttemptId1);
    when(attempt1.getRMAppAttemptMetrics()).thenReturn(attemptMetric1);
    when(app1.getCurrentAppAttempt()).thenReturn(attempt1);

    rm.getRMContext().getRMApps().put(appId1, app1);

    SchedulerEvent addAppEvent1 =
        new AppAddedSchedulerEvent(appId1, "default", "user", null, Priority.newInstance(5));
    cs.handle(addAppEvent1);
    SchedulerEvent addAttemptEvent1 = new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    cs.handle(addAttemptEvent1);
    // add app1 end

    // add app2 begin
    ApplicationId appId2 = BuilderUtils.newApplicationId(100, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(appId2, 1);

    RMAppAttemptMetrics attemptMetric2 = new RMAppAttemptMetrics(appAttemptId2, rm.getRMContext());
    RMAppImpl app2 = mock(RMAppImpl.class);
    when(app2.getApplicationId()).thenReturn(appId2);
    RMAppAttemptImpl attempt2 = mock(RMAppAttemptImpl.class);
    when(attempt2.getAppAttemptId()).thenReturn(appAttemptId2);
    when(attempt2.getRMAppAttemptMetrics()).thenReturn(attemptMetric2);
    when(app2.getCurrentAppAttempt()).thenReturn(attempt2);

    rm.getRMContext().getRMApps().put(appId2, app2);

    SchedulerEvent addAppEvent2 =
        new AppAddedSchedulerEvent(appId2, "default", "user", null, Priority.newInstance(8));
    cs.handle(addAppEvent2);
    SchedulerEvent addAttemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
    cs.handle(addAttemptEvent2);
    // add app end

    // Now, the first assignment will be for app2 since app2 is of highest
    // priority
    assertEquals(q.getApplications().size(), 2);
    assertEquals(q.getApplications().iterator().next().getApplicationAttemptId(), appAttemptId2);

    rm.stop();
  }
  @Test(timeout = 2000000)
  public void testUpdateHeartbeatResponseForCleanup() throws IOException {
    RMNodeImpl node = getRunningNode();
    NodeId nodeId = node.getNodeID();

    int rpcID = HopYarnAPIUtilities.getRPCID();
    byte[] allNMRequestData = new byte[1];
    allNMRequestData[0] = 0xA;
    try {
      RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.RegisterNM, allNMRequestData);
    } catch (IOException ex) {
      Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex);
    }
    TransactionState ts = new TransactionStateImpl(TransactionType.RM);

    // Expire a container
    ContainerId completedContainerId =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    node.handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId, ts));
    ts.decCounter(TransactionState.TransactionType.INIT);
    Assert.assertEquals(1, node.getContainersToCleanUp().size());

    rpcID = HopYarnAPIUtilities.getRPCID();
    allNMRequestData = new byte[1];
    allNMRequestData[0] = 0xA;
    try {
      RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.FinishApplicationMaster, allNMRequestData);
    } catch (IOException ex) {
      Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex);
    }
    TransactionState ts2 =
        new TransactionStateImpl(TransactionType.RM); // TransactionStateRM.newInstance(rpcID);

    // Finish an application
    ApplicationId finishedAppId = BuilderUtils.newApplicationId(0, 1);
    node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId, ts2));
    ts2.decCounter(TransactionState.TransactionType.INIT);
    Assert.assertEquals(1, node.getAppsToCleanup().size());
    rpcID = HopYarnAPIUtilities.getRPCID();
    allNMRequestData = new byte[1];
    allNMRequestData[0] = 0xA;
    try {
      RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.FinishApplicationMaster, allNMRequestData);
    } catch (IOException ex) {
      Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex);
    }
    TransactionState ts3 =
        new TransactionStateImpl(TransactionType.RM); // TransactionStateRM.newInstance(rpcID);

    // Verify status update does not clear containers/apps to cleanup
    // but updating heartbeat response for cleanup does
    RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent();
    RMNodeStatusEvent se =
        new RMNodeStatusEvent(
            statusEvent.getNodeId(),
            statusEvent.getNodeHealthStatus(),
            statusEvent.getContainers(),
            statusEvent.getKeepAliveAppIds(),
            statusEvent.getLatestResponse(),
            ts3);
    node.handle(se);

    ts3.decCounter(TransactionState.TransactionType.INIT);

    Assert.assertEquals(1, node.getContainersToCleanUp().size());
    Assert.assertEquals(1, node.getAppsToCleanup().size());
    NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class);
    node.updateNodeHeartbeatResponseForCleanup(hbrsp, ts);
    Assert.assertEquals(0, node.getContainersToCleanUp().size());
    Assert.assertEquals(0, node.getAppsToCleanup().size());
    Assert.assertEquals(1, hbrsp.getContainersToCleanup().size());
    Assert.assertEquals(completedContainerId, hbrsp.getContainersToCleanup().get(0));
    Assert.assertEquals(1, hbrsp.getApplicationsToCleanup().size());
    Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup().get(0));
  }
  @Test(timeout = 500000)
  public void testContainerUpdate() throws InterruptedException, IOException {
    // Start the node
    node.handle(
        new RMNodeEvent(
            null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    Configuration yarnconf = new YarnConfiguration();
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeAddedSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM)));
    }
    NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
    RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, "test", 0, 0, null, null, null);
    node2.handle(
        new RMNodeEvent(
            null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeAddedSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
    }
    ContainerId completedContainerIdFromNode1 =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    ContainerId completedContainerIdFromNode2_1 =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1);
    ContainerId completedContainerIdFromNode2_2 =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 2);

    RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent();
    RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent();
    RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent();

    ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
    ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
    ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);

    doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode1))
        .when(statusEventFromNode1)
        .getContainers();
    doReturn(ContainerState.COMPLETE).when(containerStatusFromNode1).getState();
    node.handle(statusEventFromNode1);
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeUpdateSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM)));
    }
    // ts.decCounter("test");
    Assert.assertEquals(1, completedContainers.size());
    Assert.assertEquals(completedContainerIdFromNode1, completedContainers.get(0).getContainerId());

    completedContainers.clear();

    doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode2_1))
        .when(statusEventFromNode2_1)
        .getContainers();
    doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_1).getState();

    doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode2_2))
        .when(statusEventFromNode2_2)
        .getContainers();
    doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_2).getState();

    node2.setNextHeartBeat(false);
    node2.handle(statusEventFromNode2_1);
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
    }

    node2.setNextHeartBeat(true);
    node2.handle(statusEventFromNode2_2);
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
    }
    // ts2.decCounter("test");
    Assert.assertEquals(2, completedContainers.size());
    Assert.assertEquals(
        completedContainerIdFromNode2_1, completedContainers.get(0).getContainerId());
    Assert.assertEquals(
        completedContainerIdFromNode2_2, completedContainers.get(1).getContainerId());
  }