コード例 #1
0
 /**
  * Check if a task can be killed to increase free memory
  *
  * @param tid task attempt ID
  * @return true if the task can be killed
  */
 private boolean isKillable(TaskAttemptID tid) {
   TaskInProgress tip = taskTracker.runningTasks.get(tid);
   return tip != null
       && !tip.wasKilled()
       && (tip.getRunState() == TaskStatus.State.RUNNING
           || tip.getRunState() == TaskStatus.State.COMMIT_PENDING);
 }
コード例 #2
0
  private void killTasksWithLeastProgress(long memoryStillInUsage) {

    List<TaskAttemptID> tasksToKill = new ArrayList<TaskAttemptID>();
    List<TaskAttemptID> tasksToExclude = new ArrayList<TaskAttemptID>();
    // Find tasks to kill so as to get memory usage under limits.
    while (memoryStillInUsage > maxMemoryAllowedForAllTasks) {
      // Exclude tasks that are already marked for
      // killing.
      TaskInProgress task = taskTracker.findTaskToKill(tasksToExclude);
      if (task == null) {
        break; // couldn't find any more tasks to kill.
      }

      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
      }
      // Exclude this task from next search because it is already
      // considered.
      tasksToExclude.add(tid);
    }

    // Now kill the tasks.
    if (!tasksToKill.isEmpty()) {
      for (TaskAttemptID tid : tasksToKill) {
        String msg =
            "Killing one of the least progress tasks - "
                + tid
                + ", as the cumulative memory usage of all the tasks on "
                + "the TaskTracker exceeds virtual memory limit "
                + maxMemoryAllowedForAllTasks
                + ".";
        LOG.warn(msg);
        killTask(tid, msg, false);
      }
    } else {
      LOG.info(
          "The total memory usage is overflowing TTs limits. "
              + "But found no alive task to kill for freeing memory.");
    }
  }