@Test
public void testNormalContainerAllocationWhenDNSUnavailable() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
YarnAPIStorageFactory.setConfiguration(conf);
RMStorageFactory.setConfiguration(conf);
MockRM rm1 = new MockRM(conf);
try {
rm1.start();
MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
RMApp app1 = rm1.submitApp(200);
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
// acquire the container.
SecurityUtilTestHelper.setTokenServiceUseIp(true);
List<Container> containers =
am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
.getAllocatedContainers();
// not able to fetch the container;
Assert.assertEquals(0, containers.size());
SecurityUtilTestHelper.setTokenServiceUseIp(false);
containers =
am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
.getAllocatedContainers();
// should be able to fetch the container;
Assert.assertEquals(1, containers.size());
} finally {
rm1.stop();
}
}
public static ContainerId newContainerId(
RecordFactory recordFactory,
ApplicationId appId,
ApplicationAttemptId appAttemptId,
int containerId) {
return ContainerId.newInstance(appAttemptId, containerId);
}
// This is to test container tokens are generated when the containers are
// acquired by the AM, not when the containers are allocated
@Test
public void testContainerTokenGeneratedOnPullRequest() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
YarnAPIStorageFactory.setConfiguration(conf);
RMStorageFactory.setConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
MockRM rm1 = new MockRM(conf);
try {
rm1.start();
MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
RMApp app1 = rm1.submitApp(200);
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
RMContainer container = rm1.getResourceScheduler().getRMContainer(containerId2);
// no container token is generated.
Assert.assertEquals(containerId2, container.getContainerId());
Assert.assertNull(container.getContainer().getContainerToken());
// acquire the container.
List<Container> containers =
am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
.getAllocatedContainers();
Assert.assertEquals(containerId2, containers.get(0).getId());
// container token is generated.
Assert.assertNotNull(containers.get(0).getContainerToken());
} finally {
rm1.stop();
}
}
private RMContainer createRMContainer(ApplicationAttemptId appAttId, int id, Resource resource) {
ContainerId containerId = ContainerId.newInstance(appAttId, id);
RMContainer rmContainer = mock(RMContainer.class);
Container container = mock(Container.class);
when(container.getResource()).thenReturn(resource);
when(container.getNodeId()).thenReturn(nodeId);
when(rmContainer.getContainer()).thenReturn(container);
when(rmContainer.getContainerId()).thenReturn(containerId);
return rmContainer;
}
@Before
public void setup() {
applicationId = ApplicationId.newInstance(9999l, 1);
applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
tezDAGID = TezDAGID.getInstance(applicationId, random.nextInt());
tezVertexID = TezVertexID.getInstance(tezDAGID, random.nextInt());
tezTaskID = TezTaskID.getInstance(tezVertexID, random.nextInt());
tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, random.nextInt());
dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
containerId = ContainerId.newInstance(applicationAttemptId, 111);
nodeId = NodeId.newInstance("node", 13435);
}
public TestMRApp(
ApplicationAttemptId applicationAttemptId,
ContainerAllocator allocator,
int maxAppAttempts) {
super(
applicationAttemptId,
ContainerId.newInstance(applicationAttemptId, 1),
"testhost",
2222,
3333,
System.currentTimeMillis(),
maxAppAttempts);
this.allocator = allocator;
this.successfullyUnregistered.set(true);
}
public WrappedContainer(boolean shouldProfile, String profileString) {
applicationID = ApplicationId.newInstance(rmIdentifier, 1);
appAttemptID = ApplicationAttemptId.newInstance(applicationID, 1);
containerID = ContainerId.newInstance(appAttemptID, 1);
nodeID = NodeId.newInstance("host", 12500);
nodeHttpAddress = "host:12501";
resource = Resource.newInstance(1024, 1);
priority = Priority.newInstance(1);
container =
Container.newInstance(containerID, nodeID, nodeHttpAddress, resource, priority, null);
chh = mock(ContainerHeartbeatHandler.class);
InetSocketAddress addr = new InetSocketAddress("localhost", 0);
tal = mock(TaskAttemptListener.class);
doReturn(addr).when(tal).getAddress();
dagID = TezDAGID.getInstance(applicationID, 1);
vertexID = TezVertexID.getInstance(dagID, 1);
taskID = TezTaskID.getInstance(vertexID, 1);
taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
eventHandler = mock(EventHandler.class);
historyEventHandler = mock(HistoryEventHandler.class);
Configuration conf = new Configuration(false);
appContext = mock(AppContext.class);
doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext).getApplicationACLs();
doReturn(eventHandler).when(appContext).getEventHandler();
doReturn(appAttemptID).when(appContext).getApplicationAttemptId();
doReturn(applicationID).when(appContext).getApplicationID();
doReturn(new SystemClock()).when(appContext).getClock();
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(conf).when(appContext).getAMConf();
mockDAGID();
taskSpec = mock(TaskSpec.class);
doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
amContainer =
new AMContainerImpl(container, chh, tal, new ContainerContextMatcher(), appContext);
}
public static ContainerId makeContainerId(long ts, int appId, int attemptId, int id) {
return ContainerId.newInstance(
ApplicationAttemptId.newInstance(ApplicationId.newInstance(ts, appId), attemptId), id);
}
public static ContainerId newContainerId(ApplicationAttemptId appAttemptId, int containerId) {
return ContainerId.newInstance(appAttemptId, containerId);
}
private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
ContainerId containerId = ContainerId.newInstance(appAttemptId, containerIdx);
return containerId;
}