@SuppressWarnings("rawtypes")
@Test
public void testNodeFailedAtCompletedMultipleSuccessfulTAs() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.taskAttemptSucceeded(wc.taskAttemptID);
TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
wc.pullTaskToRun();
wc.taskAttemptSucceeded(taID2);
wc.stopRequest();
wc.nmStopSent();
wc.containerCompleted(false);
wc.verifyState(AMContainerState.COMPLETED);
wc.nodeFailed();
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
verifyUnOrderedOutgoingEventTypes(
outgoingEvents, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_NODE_FAILED);
assertNull(wc.amContainer.getRunningTaskAttempt());
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
@Test
public void testDuplicateCompletedEvents() {
WrappedContainer wc = new WrappedContainer();
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.taskAttemptSucceeded(wc.taskAttemptID);
TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
wc.pullTaskToRun();
wc.taskAttemptSucceeded(taID2);
wc.stopRequest();
wc.nmStopSent();
wc.containerCompleted(false);
wc.verifyState(AMContainerState.COMPLETED);
wc.verifyNoOutgoingEvents();
wc.containerCompleted(false);
wc.verifyNoOutgoingEvents();
wc.verifyHistoryStopEvent();
}
@Test
public void testLocalResourceAddition() {
WrappedContainer wc = new WrappedContainer();
String rsrc1 = "rsrc1";
String rsrc2 = "rsrc2";
String rsrc3 = "rsrc3";
Map<String, LocalResource> initialResources = Maps.newHashMap();
initialResources.put(rsrc1, createLocalResource(rsrc1));
wc.launchContainer(initialResources, new Credentials());
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
AMContainerTask task1 = wc.pullTaskToRun();
assertEquals(0, task1.getAdditionalResources().size());
wc.taskAttemptSucceeded(wc.taskAttemptID);
// Add some resources to the next task.
Map<String, LocalResource> additionalResources = Maps.newHashMap();
additionalResources.put(rsrc2, createLocalResource(rsrc2));
additionalResources.put(rsrc3, createLocalResource(rsrc3));
TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taID2, additionalResources, new Credentials());
AMContainerTask task2 = wc.pullTaskToRun();
Map<String, LocalResource> pullTaskAdditionalResources = task2.getAdditionalResources();
assertEquals(2, pullTaskAdditionalResources.size());
pullTaskAdditionalResources.remove(rsrc2);
pullTaskAdditionalResources.remove(rsrc3);
assertEquals(0, pullTaskAdditionalResources.size());
wc.taskAttemptSucceeded(taID2);
// Verify Resources registered for this container.
Map<String, LocalResource> containerLRs =
new HashMap<String, LocalResource>(wc.amContainer.containerLocalResources);
assertEquals(3, containerLRs.size());
containerLRs.remove(rsrc1);
containerLRs.remove(rsrc2);
containerLRs.remove(rsrc3);
assertEquals(0, containerLRs.size());
// Try launching another task with the same reosurces as Task2. Verify the
// task is not asked to re-localize again.
TezTaskAttemptID taID3 = TezTaskAttemptID.getInstance(wc.taskID, 3);
wc.assignTaskAttempt(taID3, new HashMap<String, LocalResource>(), new Credentials());
AMContainerTask task3 = wc.pullTaskToRun();
assertEquals(0, task3.getAdditionalResources().size());
wc.taskAttemptSucceeded(taID3);
// Verify references are cleared after a container completes.
wc.containerCompleted(false);
assertNull(wc.amContainer.containerLocalResources);
assertNull(wc.amContainer.additionalLocalResources);
}
@SuppressWarnings("rawtypes")
@Test
public void testTaskAssignedToCompletedContainer() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.taskAttemptSucceeded(wc.taskAttemptID);
wc.containerCompleted(false);
wc.verifyState(AMContainerState.COMPLETED);
TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
verifyUnOrderedOutgoingEventTypes(outgoingEvents, TaskAttemptEventType.TA_CONTAINER_TERMINATED);
TaskAttemptEventContainerTerminated ctEvent =
(TaskAttemptEventContainerTerminated) outgoingEvents.get(0);
assertEquals(taID2, ctEvent.getTaskAttemptID());
wc.verifyHistoryStopEvent();
// Allocation to a completed Container is considered an error.
// TODO Is this valid ?
assertTrue(wc.amContainer.isInErrorState());
}
@SuppressWarnings("rawtypes")
@Test
public void testContainerDiskFailedAtRunning() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
wc.launchContainer();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.containerLaunched();
wc.pullTaskToRun();
wc.verifyState(AMContainerState.RUNNING);
wc.containerCompleted(ContainerExitStatus.DISKS_FAILED);
wc.verifyState(AMContainerState.COMPLETED);
verify(wc.tal).registerRunningContainer(wc.containerID);
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).register(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
verifyUnOrderedOutgoingEventTypes(
outgoingEvents, TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM);
assertFalse(wc.amContainer.isInErrorState());
// Pending task complete. (Ideally, container should be dead at this point
// and this event should not be generated. Network timeout on NM-RM heartbeat
// can cause it to be genreated)
wc.taskAttemptSucceeded(wc.taskAttemptID);
wc.verifyNoOutgoingEvents();
wc.verifyHistoryStopEvent();
assertFalse(wc.amContainer.isInErrorState());
}
@SuppressWarnings("rawtypes")
@Test
public void testContainerTimedOutAtRunning() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.verifyState(AMContainerState.RUNNING);
wc.containerTimedOut();
wc.verifyState(AMContainerState.STOP_REQUESTED);
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
// 1 to TA, 1 for RM de-allocate.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
verifyUnOrderedOutgoingEventTypes(
outgoingEvents,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
// TODO Should this be an RM DE-ALLOCATE instead ?
wc.containerCompleted(false);
wc.verifyHistoryStopEvent();
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
verifyUnOrderedOutgoingEventTypes(outgoingEvents, TaskAttemptEventType.TA_CONTAINER_TERMINATED);
assertFalse(wc.amContainer.isInErrorState());
assertNull(wc.amContainer.getRunningTaskAttempt());
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
}
@SuppressWarnings("rawtypes")
@Test
public void testNodeFailedAtIdleMultipleAttempts() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.taskAttemptSucceeded(wc.taskAttemptID);
wc.verifyState(AMContainerState.IDLE);
TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
wc.pullTaskToRun();
wc.taskAttemptSucceeded(taID2);
wc.verifyState(AMContainerState.IDLE);
wc.nodeFailed();
// Expecting a complete event from the RM
wc.verifyState(AMContainerState.STOPPING);
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
verifyUnOrderedOutgoingEventTypes(
outgoingEvents,
TaskAttemptEventType.TA_NODE_FAILED,
TaskAttemptEventType.TA_NODE_FAILED,
AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
for (Event event : outgoingEvents) {
if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
}
}
assertFalse(wc.amContainer.isInErrorState());
wc.containerCompleted(false);
wc.verifyNoOutgoingEvents();
wc.verifyHistoryStopEvent();
assertNull(wc.amContainer.getRunningTaskAttempt());
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
@Test
public void testTaskPullAtLaunching() {
WrappedContainer wc = new WrappedContainer();
wc.launchContainer();
AMContainerTask pulledTask = wc.pullTaskToRun();
wc.verifyState(AMContainerState.LAUNCHING);
wc.verifyNoOutgoingEvents();
assertFalse(pulledTask.shouldDie());
assertNull(pulledTask.getTask());
}
@Test
// Assign before launch.
public void tetSingleSuccessfulTaskFlow() {
WrappedContainer wc = new WrappedContainer();
wc.verifyState(AMContainerState.ALLOCATED);
// Launch request.
wc.launchContainer();
wc.verifyState(AMContainerState.LAUNCHING);
// 1 Launch request.
wc.verifyCountAndGetOutgoingEvents(1);
// Assign task.
wc.assignTaskAttempt(wc.taskAttemptID);
wc.verifyState(AMContainerState.LAUNCHING);
wc.verifyNoOutgoingEvents();
assertEquals(wc.taskAttemptID, wc.amContainer.getQueuedTaskAttempts().get(0));
// Container Launched
wc.containerLaunched();
wc.verifyState(AMContainerState.IDLE);
wc.verifyNoOutgoingEvents();
assertEquals(wc.taskAttemptID, wc.amContainer.getQueuedTaskAttempts().get(0));
assertNull(wc.amContainer.getRunningTaskAttempt());
verify(wc.tal).registerRunningContainer(wc.containerID);
verify(wc.chh).register(wc.containerID);
// Pull TA
AMContainerTask pulledTask = wc.pullTaskToRun();
wc.verifyState(AMContainerState.RUNNING);
wc.verifyNoOutgoingEvents();
assertFalse(pulledTask.shouldDie());
assertEquals(wc.taskSpec.getTaskAttemptID(), pulledTask.getTask().getTaskAttemptID());
assertEquals(wc.taskAttemptID, wc.amContainer.getRunningTaskAttempt());
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
wc.taskAttemptSucceeded(wc.taskAttemptID);
wc.verifyState(AMContainerState.IDLE);
wc.verifyNoOutgoingEvents();
assertNull(wc.amContainer.getRunningTaskAttempt());
verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID);
wc.containerCompleted(false);
wc.verifyHistoryStopEvent();
wc.verifyState(AMContainerState.COMPLETED);
wc.verifyNoOutgoingEvents();
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
assertFalse(wc.amContainer.isInErrorState());
}
@SuppressWarnings("rawtypes")
@Test
public void testAllocationAtRunning() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.verifyState(AMContainerState.RUNNING);
TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
wc.verifyState(AMContainerState.STOP_REQUESTED);
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
// 1 for NM stop request. 2 TERMINATING to TaskAttempt.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
verifyUnOrderedOutgoingEventTypes(
outgoingEvents,
NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_CONTAINER_TERMINATING);
assertTrue(wc.amContainer.isInErrorState());
wc.nmStopSent();
wc.containerCompleted(false);
wc.verifyHistoryStopEvent();
// 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
verifyUnOrderedOutgoingEventTypes(
outgoingEvents,
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
TaskAttemptEventType.TA_CONTAINER_TERMINATED);
assertNull(wc.amContainer.getRunningTaskAttempt());
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
@Test
public void testSingleSuccessfulTaskFlowFailedNMStopRequest() {
WrappedContainer wc = new WrappedContainer();
wc.verifyState(AMContainerState.ALLOCATED);
wc.launchContainer();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.containerLaunched();
wc.pullTaskToRun();
wc.taskAttemptSucceeded(wc.taskAttemptID);
wc.stopRequest();
wc.verifyState(AMContainerState.STOP_REQUESTED);
// Event to NM to stop the container.
wc.verifyCountAndGetOutgoingEvents(1);
assertTrue(
wc.verifyCountAndGetOutgoingEvents(1).get(0).getType()
== NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
wc.nmStopFailed();
wc.verifyState(AMContainerState.STOPPING);
// Event to ask a RM container release.
wc.verifyCountAndGetOutgoingEvents(1);
assertTrue(
wc.verifyCountAndGetOutgoingEvents(1).get(0).getType()
== AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
wc.containerCompleted(false);
wc.verifyHistoryStopEvent();
wc.verifyState(AMContainerState.COMPLETED);
wc.verifyNoOutgoingEvents();
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
assertNull(wc.amContainer.getRunningTaskAttempt());
assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
assertFalse(wc.amContainer.isInErrorState());
}