Exemplo n.º 1
0
  // This is to test container tokens are generated when the containers are
  // acquired by the AM, not when the containers are allocated
  @Test
  public void testContainerTokenGeneratedOnPullRequest() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm1 = new MockRM(conf);
    try {
      rm1.start();

      MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
      RMApp app1 = rm1.submitApp(200);
      MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
      // request a container.
      am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
      ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
      rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

      RMContainer container = rm1.getResourceScheduler().getRMContainer(containerId2);
      // no container token is generated.
      Assert.assertEquals(containerId2, container.getContainerId());
      Assert.assertNull(container.getContainer().getContainerToken());

      // acquire the container.
      List<Container> containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      Assert.assertEquals(containerId2, containers.get(0).getId());
      // container token is generated.
      Assert.assertNotNull(containers.get(0).getContainerToken());
    } finally {
      rm1.stop();
    }
  }
Exemplo n.º 2
0
  @Test
  public void testNormalContainerAllocationWhenDNSUnavailable() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    MockRM rm1 = new MockRM(conf);
    try {
      rm1.start();
      MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
      RMApp app1 = rm1.submitApp(200);
      MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

      // request a container.
      am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
      ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
      rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

      // acquire the container.
      SecurityUtilTestHelper.setTokenServiceUseIp(true);
      List<Container> containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      // not able to fetch the container;
      Assert.assertEquals(0, containers.size());

      SecurityUtilTestHelper.setTokenServiceUseIp(false);
      containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      // should be able to fetch the container;
      Assert.assertEquals(1, containers.size());
    } finally {
      rm1.stop();
    }
  }
  @Test(timeout = 30000)
  public void testNodeUpdate() throws Exception {
    // set node -> label
    mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));

    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));

    // inject node label manager
    MockRM rm =
        new MockRM(getConfigurationWithQueueLabels(conf)) {
          @Override
          public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
          }
        };

    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 8000);
    MockNM nm2 = rm.registerNode("h2:1234", 8000);
    MockNM nm3 = rm.registerNode("h3:1234", 8000);

    ContainerId containerId;

    // launch an app to queue a1 (label = x), and check all container will
    // be allocated in h1
    RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);

    // request a container.
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED, 10 * 1000));

    // check used resource:
    // queue-a used x=1G, ""=1G
    checkUsedResource(rm, "a", 1024, "x");
    checkUsedResource(rm, "a", 1024);

    // change h1's label to z, container should be killed
    mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("z")));
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.KILLED, 10 * 1000));

    // check used resource:
    // queue-a used x=0G, ""=1G ("" not changed)
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 1024);

    // request a container with label = y
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "y");
    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    Assert.assertTrue(rm.waitForState(nm2, containerId, RMContainerState.ALLOCATED, 10 * 1000));

    // check used resource:
    // queue-a used y=1G, ""=1G
    checkUsedResource(rm, "a", 1024, "y");
    checkUsedResource(rm, "a", 1024);

    // change h2's label to no label, container should be killed
    mgr.replaceLabelsOnNode(
        ImmutableMap.of(NodeId.newInstance("h2", 0), CommonNodeLabelsManager.EMPTY_STRING_SET));
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.KILLED, 10 * 1000));

    // check used resource:
    // queue-a used x=0G, y=0G, ""=1G ("" not changed)
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 0, "y");
    checkUsedResource(rm, "a", 1024);

    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);

    // change h3's label to z, AM container should be killed
    mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h3", 0), toSet("z")));
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.KILLED, 10 * 1000));

    // check used resource:
    // queue-a used x=0G, y=0G, ""=1G ("" not changed)
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 0, "y");
    checkUsedResource(rm, "a", 0);

    rm.close();
  }