@Test
  public void testNormalContainerAllocationWhenDNSUnavailable() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    MockRM rm1 = new MockRM(conf);
    try {
      rm1.start();
      MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
      RMApp app1 = rm1.submitApp(200);
      MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

      // request a container.
      am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
      ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
      rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

      // acquire the container.
      SecurityUtilTestHelper.setTokenServiceUseIp(true);
      List<Container> containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      // not able to fetch the container;
      Assert.assertEquals(0, containers.size());

      SecurityUtilTestHelper.setTokenServiceUseIp(false);
      containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      // should be able to fetch the container;
      Assert.assertEquals(1, containers.size());
    } finally {
      rm1.stop();
    }
  }
Beispiel #2
0
 public static ContainerId newContainerId(
     RecordFactory recordFactory,
     ApplicationId appId,
     ApplicationAttemptId appAttemptId,
     int containerId) {
   return ContainerId.newInstance(appAttemptId, containerId);
 }
  // This is to test container tokens are generated when the containers are
  // acquired by the AM, not when the containers are allocated
  @Test
  public void testContainerTokenGeneratedOnPullRequest() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    YarnAPIStorageFactory.setConfiguration(conf);
    RMStorageFactory.setConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm1 = new MockRM(conf);
    try {
      rm1.start();

      MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
      RMApp app1 = rm1.submitApp(200);
      MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
      // request a container.
      am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
      ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
      rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

      RMContainer container = rm1.getResourceScheduler().getRMContainer(containerId2);
      // no container token is generated.
      Assert.assertEquals(containerId2, container.getContainerId());
      Assert.assertNull(container.getContainer().getContainerToken());

      // acquire the container.
      List<Container> containers =
          am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
              .getAllocatedContainers();
      Assert.assertEquals(containerId2, containers.get(0).getId());
      // container token is generated.
      Assert.assertNotNull(containers.get(0).getContainerToken());
    } finally {
      rm1.stop();
    }
  }
 private RMContainer createRMContainer(ApplicationAttemptId appAttId, int id, Resource resource) {
   ContainerId containerId = ContainerId.newInstance(appAttId, id);
   RMContainer rmContainer = mock(RMContainer.class);
   Container container = mock(Container.class);
   when(container.getResource()).thenReturn(resource);
   when(container.getNodeId()).thenReturn(nodeId);
   when(rmContainer.getContainer()).thenReturn(container);
   when(rmContainer.getContainerId()).thenReturn(containerId);
   return rmContainer;
 }
 @Before
 public void setup() {
   applicationId = ApplicationId.newInstance(9999l, 1);
   applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
   tezDAGID = TezDAGID.getInstance(applicationId, random.nextInt());
   tezVertexID = TezVertexID.getInstance(tezDAGID, random.nextInt());
   tezTaskID = TezTaskID.getInstance(tezVertexID, random.nextInt());
   tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, random.nextInt());
   dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
   containerId = ContainerId.newInstance(applicationAttemptId, 111);
   nodeId = NodeId.newInstance("node", 13435);
 }
 public TestMRApp(
     ApplicationAttemptId applicationAttemptId,
     ContainerAllocator allocator,
     int maxAppAttempts) {
   super(
       applicationAttemptId,
       ContainerId.newInstance(applicationAttemptId, 1),
       "testhost",
       2222,
       3333,
       System.currentTimeMillis(),
       maxAppAttempts);
   this.allocator = allocator;
   this.successfullyUnregistered.set(true);
 }
Beispiel #7
0
    public WrappedContainer(boolean shouldProfile, String profileString) {
      applicationID = ApplicationId.newInstance(rmIdentifier, 1);
      appAttemptID = ApplicationAttemptId.newInstance(applicationID, 1);
      containerID = ContainerId.newInstance(appAttemptID, 1);
      nodeID = NodeId.newInstance("host", 12500);
      nodeHttpAddress = "host:12501";
      resource = Resource.newInstance(1024, 1);
      priority = Priority.newInstance(1);
      container =
          Container.newInstance(containerID, nodeID, nodeHttpAddress, resource, priority, null);

      chh = mock(ContainerHeartbeatHandler.class);

      InetSocketAddress addr = new InetSocketAddress("localhost", 0);
      tal = mock(TaskAttemptListener.class);
      doReturn(addr).when(tal).getAddress();

      dagID = TezDAGID.getInstance(applicationID, 1);
      vertexID = TezVertexID.getInstance(dagID, 1);
      taskID = TezTaskID.getInstance(vertexID, 1);
      taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);

      eventHandler = mock(EventHandler.class);
      historyEventHandler = mock(HistoryEventHandler.class);

      Configuration conf = new Configuration(false);
      appContext = mock(AppContext.class);
      doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext).getApplicationACLs();
      doReturn(eventHandler).when(appContext).getEventHandler();
      doReturn(appAttemptID).when(appContext).getApplicationAttemptId();
      doReturn(applicationID).when(appContext).getApplicationID();
      doReturn(new SystemClock()).when(appContext).getClock();
      doReturn(historyEventHandler).when(appContext).getHistoryHandler();
      doReturn(conf).when(appContext).getAMConf();
      mockDAGID();

      taskSpec = mock(TaskSpec.class);
      doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();

      amContainer =
          new AMContainerImpl(container, chh, tal, new ContainerContextMatcher(), appContext);
    }
 public static ContainerId makeContainerId(long ts, int appId, int attemptId, int id) {
   return ContainerId.newInstance(
       ApplicationAttemptId.newInstance(ApplicationId.newInstance(ts, appId), attemptId), id);
 }
Beispiel #9
0
 public static ContainerId newContainerId(ApplicationAttemptId appAttemptId, int containerId) {
   return ContainerId.newInstance(appAttemptId, containerId);
 }
 private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
   ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
   ContainerId containerId = ContainerId.newInstance(appAttemptId, containerIdx);
   return containerId;
 }