/** Process incoming heartbeat messages from the task trackers. */
public synchronized int emitHeartbeat(TaskTrackerStatus trackerStatus, boolean initialContact) {
String trackerName = trackerStatus.getTrackerName();
trackerStatus.setLastSeen(System.currentTimeMillis());
synchronized (taskTrackers) {
synchronized (trackerExpiryQueue) {
boolean seenBefore = updateTaskTrackerStatus(trackerName, trackerStatus);
if (initialContact) {
// If it's first contact, then clear out any state hanging around
if (seenBefore) {
lostTaskTracker(trackerName);
}
} else {
// If not first contact, there should be some record of the tracker
if (!seenBefore) {
return InterTrackerProtocol.UNKNOWN_TASKTRACKER;
}
}
if (initialContact) {
trackerExpiryQueue.add(trackerStatus);
}
}
}
updateTaskStatuses(trackerStatus);
// LOG.info("Got heartbeat from "+trackerName);
return InterTrackerProtocol.TRACKERS_OK;
}
void removeTaskEntry(String taskid) {
// taskid --> tracker
String tracker = (String) taskidToTrackerMap.remove(taskid);
// tracker --> taskid
TreeSet trackerSet = (TreeSet) trackerToTaskMap.get(tracker);
if (trackerSet != null) {
trackerSet.remove(taskid);
}
// taskid --> TIP
taskidToTIPMap.remove(taskid);
}
/**
* We lost the task tracker! All task-tracker structures have already been updated. Just process
* the contained tasks and any jobs that might be affected.
*/
void lostTaskTracker(String trackerName) {
LOG.info("Lost tracker '" + trackerName + "'");
TreeSet lostTasks = (TreeSet) trackerToTaskMap.get(trackerName);
trackerToTaskMap.remove(trackerName);
if (lostTasks != null) {
for (Iterator it = lostTasks.iterator(); it.hasNext(); ) {
String taskId = (String) it.next();
TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
// Tell the job to fail the relevant task
JobInProgress job = tip.getJob();
job.failedTask(tip, taskId, trackerName);
}
}
}
/** Return a table of block data */
public Block[] getBlockReport() {
TreeSet<Block> blockSet = new TreeSet<Block>();
volumes.getBlockInfo(blockSet);
Block blockTable[] = new Block[blockSet.size()];
int i = 0;
for (Iterator<Block> it = blockSet.iterator(); it.hasNext(); i++) {
blockTable[i] = it.next();
}
return blockTable;
}
/**
* A tracker wants to know if any of its Tasks have been closed (because the job completed,
* whether successfully or not)
*/
public synchronized String pollForTaskWithClosedJob(String taskTracker) {
TreeSet taskIds = (TreeSet) trackerToTaskMap.get(taskTracker);
if (taskIds != null) {
for (Iterator it = taskIds.iterator(); it.hasNext(); ) {
String taskId = (String) it.next();
TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
if (tip.shouldCloseForClosedJob(taskId)) {
//
// This is how the JobTracker ends a task at the TaskTracker.
// It may be successfully completed, or may be killed in
// mid-execution.
//
return taskId;
}
}
}
return null;
}
/** Populate the given blockSet with any child blocks found at this node. */
public void getBlockInfo(TreeSet<Block> blockSet) {
if (children != null) {
for (int i = 0; i < children.length; i++) {
children[i].getBlockInfo(blockSet);
}
}
File blockFiles[] = dir.listFiles();
for (int i = 0; i < blockFiles.length; i++) {
if (Block.isBlockFilename(blockFiles[i])) {
long genStamp = getGenerationStampFromFile(blockFiles, blockFiles[i]);
blockSet.add(new Block(blockFiles[i], blockFiles[i].length(), genStamp));
}
}
}
///////////////////////////////////////////////////////
// Maintain lookup tables; called by JobInProgress
// and TaskInProgress
///////////////////////////////////////////////////////
void createTaskEntry(String taskid, String taskTracker, TaskInProgress tip) {
LOG.info(
"Adding task '"
+ taskid
+ "' to tip "
+ tip.getTIPId()
+ ", for tracker '"
+ taskTracker
+ "'");
// taskid --> tracker
taskidToTrackerMap.put(taskid, taskTracker);
// tracker --> taskid
TreeSet taskset = (TreeSet) trackerToTaskMap.get(taskTracker);
if (taskset == null) {
taskset = new TreeSet();
trackerToTaskMap.put(taskTracker, taskset);
}
taskset.add(taskid);
// taskid --> TIP
taskidToTIPMap.put(taskid, tip);
}