private void validateResourceRequest(ApplicationSubmissionContext submissionContext)
      throws InvalidResourceRequestException {
    // Validation of the ApplicationSubmissionContext needs to be completed
    // here. Only those fields that are dependent on RM's configuration are
    // checked here as they have to be validated whether they are part of new
    // submission or just being recovered.

    // Check whether AM resource requirements are within required limits
    if (!submissionContext.getUnmanagedAM()) {
      ResourceRequest amReq =
          BuilderUtils.newResourceRequest(
              RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
              ResourceRequest.ANY,
              submissionContext.getResource(),
              1);
      try {
        SchedulerUtils.validateResourceRequest(amReq, scheduler.getMaximumResourceCapability());
      } catch (InvalidResourceRequestException e) {
        LOG.warn(
            "RM app submission failed in validating AM resource request"
                + " for application "
                + submissionContext.getApplicationId(),
            e);
        throw e;
      }
    }
  }
示例#2
0
  private ResourceRequest validateAndCreateResourceRequest(
      ApplicationSubmissionContext submissionContext, boolean isRecovery)
      throws InvalidResourceRequestException {
    // Validation of the ApplicationSubmissionContext needs to be completed
    // here. Only those fields that are dependent on RM's configuration are
    // checked here as they have to be validated whether they are part of new
    // submission or just being recovered.

    // Check whether AM resource requirements are within required limits
    if (!submissionContext.getUnmanagedAM()) {
      ResourceRequest amReq = submissionContext.getAMContainerResourceRequest();
      if (amReq == null) {
        amReq =
            BuilderUtils.newResourceRequest(
                RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
                ResourceRequest.ANY,
                submissionContext.getResource(),
                1);
      }

      // set label expression for AM container
      if (null == amReq.getNodeLabelExpression()) {
        amReq.setNodeLabelExpression(submissionContext.getNodeLabelExpression());
      }

      try {
        SchedulerUtils.normalizeAndValidateRequest(
            amReq,
            scheduler.getMaximumResourceCapability(),
            submissionContext.getQueue(),
            scheduler,
            isRecovery,
            rmContext);
      } catch (InvalidResourceRequestException e) {
        LOG.warn(
            "RM app submission failed in validating AM resource request"
                + " for application "
                + submissionContext.getApplicationId(),
            e);
        throw e;
      }

      SchedulerUtils.normalizeRequest(
          amReq,
          scheduler.getResourceCalculator(),
          scheduler.getClusterResource(),
          scheduler.getMinimumResourceCapability(),
          scheduler.getMaximumResourceCapability(),
          scheduler.getMinimumResourceCapability());
      return amReq;
    }

    return null;
  }
  @Test(timeout = 500000)
  public void testContainerUpdate() throws InterruptedException, IOException {
    // Start the node
    node.handle(
        new RMNodeEvent(
            null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    Configuration yarnconf = new YarnConfiguration();
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeAddedSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM)));
    }
    NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
    RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, "test", 0, 0, null, null, null);
    node2.handle(
        new RMNodeEvent(
            null, RMNodeEventType.STARTED, new TransactionStateImpl(TransactionType.RM)));
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeAddedSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
    }
    ContainerId completedContainerIdFromNode1 =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    ContainerId completedContainerIdFromNode2_1 =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1);
    ContainerId completedContainerIdFromNode2_2 =
        BuilderUtils.newContainerId(
            BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 2);

    RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent();
    RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent();
    RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent();

    ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
    ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
    ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);

    doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode1))
        .when(statusEventFromNode1)
        .getContainers();
    doReturn(ContainerState.COMPLETE).when(containerStatusFromNode1).getState();
    node.handle(statusEventFromNode1);
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeUpdateSchedulerEvent(node, new TransactionStateImpl(TransactionType.RM)));
    }
    // ts.decCounter("test");
    Assert.assertEquals(1, completedContainers.size());
    Assert.assertEquals(completedContainerIdFromNode1, completedContainers.get(0).getContainerId());

    completedContainers.clear();

    doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode2_1))
        .when(statusEventFromNode2_1)
        .getContainers();
    doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_1).getState();

    doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode2_2))
        .when(statusEventFromNode2_2)
        .getContainers();
    doReturn(ContainerState.COMPLETE).when(containerStatusFromNode2_2).getState();

    node2.setNextHeartBeat(false);
    node2.handle(statusEventFromNode2_1);
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
    }

    node2.setNextHeartBeat(true);
    node2.handle(statusEventFromNode2_2);
    // If Distributed RT is enabled, this is the only way to let the scheduler
    // pick up the event, the PendingEvent retrieval does not invoke the
    // Mock Scheduler
    if (yarnconf.getBoolean(
        YarnConfiguration.DISTRIBUTED_RM, YarnConfiguration.DEFAULT_DISTRIBUTED_RM)) {
      scheduler.handle(
          new NodeUpdateSchedulerEvent(node2, new TransactionStateImpl(TransactionType.RM)));
    }
    // ts2.decCounter("test");
    Assert.assertEquals(2, completedContainers.size());
    Assert.assertEquals(
        completedContainerIdFromNode2_1, completedContainers.get(0).getContainerId());
    Assert.assertEquals(
        completedContainerIdFromNode2_2, completedContainers.get(1).getContainerId());
  }