@Test
public void testUpdateShuffleNode() throws Exception {
JobId jobId = JobId.newJobId(createTestAppId(), 123456);
DragonZooKeeper.NodeData node = new DragonZooKeeper.NodeData();
node.nodeId = createTestNodeId();
node.taskId = TaskId.newTaskId(jobId, 34565, TaskType.MAP);
dragonZK.createShufflePath(jobId);
dragonZK.createShuffleNode(jobId, newArrayList(node));
node.nodeId.setHost("NewServerHost");
dragonZK.updateShuffleNode(jobId, newArrayList(node));
Stat stat =
zkClient
.checkExists()
.forPath(
"/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565");
assertNotNull(stat);
byte[] data =
zkClient
.getData()
.forPath(
"/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565");
assertEquals("NewServerHost:2345", new String(data));
}
@Test
public void testGetShufflePath() throws Exception {
ApplicationId appId = createTestAppId();
JobId jobId = JobId.newJobId(appId, 123456);
assertEquals(
"/dragon/job_458495849584_89894985_123456/shuffle", dragonZK.getShufflePath(jobId));
}
@Test
public void testGetTaskPath() throws Exception {
ApplicationId appId = createTestAppId();
JobId jobId = JobId.newJobId(appId, 123456);
TaskId taskId = TaskId.newTaskId(jobId, 34565, TaskType.MAP);
assertEquals(
"/dragon/job_458495849584_89894985_123456/shuffle/task_458495849584_89894985_123456_m_34565",
dragonZK.getTaskPath(jobId, taskId));
}
@Test
public void testCreateShufflePath() throws Exception {
ApplicationId appId = createTestAppId();
JobId jobId = JobId.newJobId(appId, 123456);
dragonZK.createShufflePath(jobId);
Stat stat = zkClient.checkExists().forPath("/dragon/job_458495849584_89894985_123456/shuffle");
assertNotNull(stat);
}
@Test
public void testGetShuffleNodeByTaskId() throws Exception {
JobId jobId = JobId.newJobId(createTestAppId(), 123456);
TaskId taskId = TaskId.newTaskId(jobId, 34565, TaskType.MAP);
DragonZooKeeper.NodeData node = new DragonZooKeeper.NodeData();
node.nodeId = createTestNodeId();
node.taskId = taskId;
if (zkClient.checkExists().forPath("/dragon/job_458495849584_89894985_123456/shuffle")
== null) {
dragonZK.createShufflePath(jobId);
}
PathChildrenCache shuffleNodeCache =
new PathChildrenCache(zkClient, dragonZK.getShufflePath(jobId), true);
dragonZK.setShuffleNodeCache(shuffleNodeCache);
dragonZK.createShuffleNode(jobId, newArrayList(node));
Thread.sleep(1000);
NodeId nodeId = dragonZK.getShuffleNodeByTaskId(jobId, taskId);
assertNotNull(nodeId);
assertEquals(node.nodeId, nodeId);
}
@Override
public void init(final Configuration conf) {
// Get all needed security tokens.
downloadTokensAndSetupUGI(conf);
// Initialize application context,name,attemptId,jobId
context = new RunningAppContext();
appName = conf.get(DragonJobConfig.JOB_NAME, "<missing app name>");
conf.setInt(DragonJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId.getAttemptId());
jobId =
JobId.newJobId(appAttemptId.getApplicationId(), appAttemptId.getApplicationId().getId());
// service to hand out event
dispatcher = createDispatcher();
addIfService(dispatcher);
// service to handle requests from JobClient
clientService = createClientService(context);
addIfService(clientService);
// service to log job history events
EventHandler<JobHistoryEvent> historyHandlerService = createJobHistoryHandler(context);
dispatcher.register(
org.apache.hadoop.realtime.jobhistory.EventType.class, historyHandlerService);
// Initialize the JobEventDispatcher
this.jobEventDispatcher = new JobEventDispatcher();
// register the event dispatchers
dispatcher.register(JobEventType.class, jobEventDispatcher);
dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher());
// service to handle requests to TaskUmbilicalProtocol
childService = createChildService(context);
addIfService(childService);
dispatcher.register(ChildExecutionEventType.class, childService);
// service to allocate containers from RM (if non-uber) or to fake it (uber)
containerAllocator = createContainerAllocator(clientService, context);
addIfService(containerAllocator);
dispatcher.register(ContainerAllocator.EventType.class, containerAllocator);
// corresponding service to launch allocated containers via NodeManager
containerLauncher = createContainerLauncher(context);
addIfService(containerLauncher);
dispatcher.register(ContainerLauncher.EventType.class, containerLauncher);
// dragon zookeeper service
zkService = new DragonZKService(context);
addIfService(zkService);
dispatcher.register(ZKEventType.class, zkService);
// Add the JobHistoryEventHandler last so that it is properly stopped first.
// This will guarantee that all history-events are flushed before AM goes
// ahead with shutdown.
// Note: Even though JobHistoryEventHandler is started last, if any
// component creates a JobHistoryEvent in the meanwhile, it will be just be
// queued inside the JobHistoryEventHandler
addIfService(historyHandlerService);
super.init(conf);
}