예제 #1
0
 @Before
 public void setup() throws IOException {
   YarnConfiguration conf = new YarnConfiguration();
   YarnAPIStorageFactory.setConfiguration(conf);
   RMStorageFactory.setConfiguration(conf);
   RMUtilities.InitializeDB();
 }
예제 #2
0
  @Test
  public void testNormalContainerAllocationWhenDNSUnavailable() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    MockRM rm1 = new MockRM(conf);
    try {
      rm1.start();
      MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
      RMApp app1 = rm1.submitApp(200);
      MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

      // request a container.
      am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
      ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
      rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

      // acquire the container.
      SecurityUtilTestHelper.setTokenServiceUseIp(true);
      List<Container> containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      // not able to fetch the container;
      Assert.assertEquals(0, containers.size());

      SecurityUtilTestHelper.setTokenServiceUseIp(false);
      containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      // should be able to fetch the container;
      Assert.assertEquals(1, containers.size());
    } finally {
      rm1.stop();
    }
  }
예제 #3
0
  // This is to test container tokens are generated when the containers are
  // acquired by the AM, not when the containers are allocated
  @Test
  public void testContainerTokenGeneratedOnPullRequest() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm1 = new MockRM(conf);
    try {
      rm1.start();

      MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
      RMApp app1 = rm1.submitApp(200);
      MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
      // request a container.
      am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
      ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
      rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

      RMContainer container = rm1.getResourceScheduler().getRMContainer(containerId2);
      // no container token is generated.
      Assert.assertEquals(containerId2, container.getContainerId());
      Assert.assertNull(container.getContainer().getContainerToken());

      // acquire the container.
      List<Container> containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      Assert.assertEquals(containerId2, containers.get(0).getId());
      // container token is generated.
      Assert.assertNotNull(containers.get(0).getContainerToken());
    } finally {
      rm1.stop();
    }
  }
예제 #4
0
  @Before
  public void setUp() throws Exception {
    InlineDispatcher rmDispatcher = new InlineDispatcher();

    YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    RMUtilities.InitializeDB();
    TransactionStateManager tsm = new TransactionStateManager();
    tsm.init(conf);
    tsm.start();
    rmContext =
        new RMContextImpl(
            rmDispatcher,
            null,
            null,
            null,
            mock(DelegationTokenRenewer.class),
            null,
            null,
            null,
            conf,
            tsm);
    NodesListManager nodesListManager = mock(NodesListManager.class);
    HostsFileReader reader = mock(HostsFileReader.class);
    when(nodesListManager.getHostsReader()).thenReturn(reader);
    ((RMContextImpl) rmContext).setNodesListManager(nodesListManager);
    scheduler = mock(YarnScheduler.class);
    doAnswer(
            new Answer<Void>() {
              @Override
              public Void answer(InvocationOnMock invocation) throws Throwable {
                final SchedulerEvent event = (SchedulerEvent) (invocation.getArguments()[0]);
                eventType = event.getType();
                if (eventType == SchedulerEventType.NODE_UPDATE) {
                  List<UpdatedContainerInfo> lastestContainersInfoList =
                      ((NodeUpdateSchedulerEvent) event)
                          .getRMNode()
                          .pullContainerUpdates(event.getTransactionState());
                  for (UpdatedContainerInfo lastestContainersInfo : lastestContainersInfoList) {
                    completedContainers.addAll(lastestContainersInfo.getCompletedContainers());
                  }
                }
                return null;
              }
            })
        .when(scheduler)
        .handle(any(SchedulerEvent.class));

    rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher());

    rmDispatcher.register(
        NodesListManagerEventType.class, new TestNodeListManagerEventDispatcher());

    NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
    node = new RMNodeImpl(nodeId, rmContext, nodeId.getHost(), 0, 0, null, null, null);
    nodesListManagerEvent = null;
  }
예제 #5
0
            @Override
            protected void configureServlets() {
              try {
                bind(JAXBContextResolver.class);
                bind(RMWebServices.class);
                bind(GenericExceptionHandler.class);
                Configuration conf = new Configuration();
                conf.setClass(
                    YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);

                YarnAPIStorageFactory.setConfiguration(conf);
                RMStorageFactory.setConfiguration(conf);
                RMStorageFactory.getConnector().formatStorage();
                rm = new MockRM(conf);
                bind(ResourceManager.class).toInstance(rm);
                bind(RMContext.class).toInstance(rm.getRMContext());
                bind(ApplicationACLsManager.class).toInstance(rm.getApplicationACLsManager());
                bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager());
                serve("/*").with(GuiceContainer.class);
              } catch (Exception ex) {
                Logger.getLogger(TestRMWebServices.class.getName()).log(Level.SEVERE, null, ex);
              }
            }
예제 #6
0
  // This is to test fetching AM container will be retried, if AM container is
  // not fetchable since DNS is unavailable causing container token/NMtoken
  // creation failure.
  @Test(timeout = 60000)
  public void testAMContainerAllocationWhenDNSUnavailable() throws Exception {
    final YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    MockRM rm1 =
        new MockRM(conf) {
          @Override
          protected RMSecretManagerService createRMSecretManagerService() {
            return new TestRMSecretManagerService(conf, rmContext);
          }
        };
    try {
      rm1.start();

      MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
      SecurityUtilTestHelper.setTokenServiceUseIp(true);
      RMApp app1 = rm1.submitApp(200);
      RMAppAttempt attempt = app1.getCurrentAppAttempt();
      nm1.nodeHeartbeat(true);

      // fetching am container will fail, keep retrying 5 times.
      while (numRetries <= 5) {
        nm1.nodeHeartbeat(true);
        Thread.sleep(1000);
        Assert.assertEquals(RMAppAttemptState.SCHEDULED, attempt.getState());
        System.out.println("Waiting for am container to be allocated.");
      }

      SecurityUtilTestHelper.setTokenServiceUseIp(false);
      rm1.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.ALLOCATED);
      MockRM.launchAndRegisterAM(app1, rm1, nm1);
    } finally {
      rm1.stop();
    }
  }
예제 #7
0
  @Test(timeout = 30000)
  public void testExcessReservationThanNodeManagerCapacity() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    MockRM rm = new MockRM(conf);
    try {
      rm.start();

      // Register node1
      MockNM nm1 = rm.registerNode("127.0.0.1:1234", 2 * GB, 4);
      MockNM nm2 = rm.registerNode("127.0.0.1:2234", 3 * GB, 4);

      nm1.nodeHeartbeat(true);
      nm2.nodeHeartbeat(true);
      // HOP :: Sleep to allow previous events to be processed
      Thread.sleep(
          conf.getInt(
                  YarnConfiguration.HOPS_PENDING_EVENTS_RETRIEVAL_PERIOD,
                  YarnConfiguration.DEFAULT_HOPS_PENDING_EVENTS_RETRIEVAL_PERIOD)
              * 2);
      // wait..
      int waitCount = 20;
      int size = rm.getRMContext().getActiveRMNodes().size();
      while ((size = rm.getRMContext().getActiveRMNodes().size()) != 2 && waitCount-- > 0) {
        LOG.info("Waiting for node managers to register : " + size);
        Thread.sleep(100);
      }
      Assert.assertEquals(2, rm.getRMContext().getActiveRMNodes().size());
      // Submit an application
      RMApp app1 = rm.submitApp(128);

      // kick the scheduling
      nm1.nodeHeartbeat(true);
      RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
      MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId(), nm1);
      am1.registerAppAttempt();

      LOG.info("sending container requests ");
      am1.addRequests(new String[] {"*"}, 3 * GB, 1, 1);
      AllocateResponse alloc1Response = am1.schedule(); // send the request

      // kick the scheduler
      nm1.nodeHeartbeat(true);
      int waitCounter = 20;
      LOG.info("heartbeating nm1");
      while (alloc1Response.getAllocatedContainers().size() < 1 && waitCounter-- > 0) {
        LOG.info("Waiting for containers to be created for app 1...");
        Thread.sleep(500);
        alloc1Response = am1.schedule();
      }
      LOG.info("received container : " + alloc1Response.getAllocatedContainers().size());

      // No container should be allocated.
      // Internally it should not been reserved.
      Assert.assertTrue(alloc1Response.getAllocatedContainers().size() == 0);

      LOG.info("heartbeating nm2");
      waitCounter = 20;
      nm2.nodeHeartbeat(true);
      while (alloc1Response.getAllocatedContainers().size() < 1 && waitCounter-- > 0) {
        LOG.info("Waiting for containers to be created for app 1...");
        Thread.sleep(500);
        alloc1Response = am1.schedule();
      }
      LOG.info("received container : " + alloc1Response.getAllocatedContainers().size());
      Assert.assertTrue(alloc1Response.getAllocatedContainers().size() == 1);
    } finally {
      rm.stop();
    }
  }