Exemplo n.º 1
0
  /**
   * A tracker wants to know if there's a Task to run. Returns a task we'd like the TaskTracker to
   * execute right now.
   *
   * <p>Eventually this function should compute load on the various TaskTrackers, and incorporate
   * knowledge of DFS file placement. But for right now, it just grabs a single item out of the
   * pending task list and hands it back.
   */
  public synchronized Task pollForNewTask(String taskTracker) {
    //
    // Compute average map and reduce task numbers across pool
    //
    int avgMaps = 0;
    int avgReduces = 0;
    int numTaskTrackers;
    TaskTrackerStatus tts;
    synchronized (taskTrackers) {
      numTaskTrackers = taskTrackers.size();
      tts = (TaskTrackerStatus) taskTrackers.get(taskTracker);
    }
    if (numTaskTrackers > 0) {
      avgMaps = totalMaps / numTaskTrackers;
      avgReduces = totalReduces / numTaskTrackers;
    }
    int totalCapacity = numTaskTrackers * maxCurrentTasks;
    //
    // Get map + reduce counts for the current tracker.
    //
    if (tts == null) {
      LOG.warning("Unknown task tracker polling; ignoring: " + taskTracker);
      return null;
    }

    int numMaps = tts.countMapTasks();
    int numReduces = tts.countReduceTasks();

    //
    // In the below steps, we allocate first a map task (if appropriate),
    // and then a reduce task if appropriate.  We go through all jobs
    // in order of job arrival; jobs only get serviced if their
    // predecessors are serviced, too.
    //

    //
    // We hand a task to the current taskTracker if the given machine
    // has a workload that's equal to or less than the averageMaps
    // +/- TASK_ALLOC_EPSILON.  (That epsilon is in place in case
    // there is an odd machine that is failing for some reason but
    // has not yet been removed from the pool, making capacity seem
    // larger than it really is.)
    //
    synchronized (jobsByArrival) {
      if ((numMaps < maxCurrentTasks) && (numMaps <= (avgMaps + TASK_ALLOC_EPSILON))) {

        int totalNeededMaps = 0;
        for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
          JobInProgress job = (JobInProgress) it.next();
          if (job.getStatus().getRunState() != JobStatus.RUNNING) {
            continue;
          }

          Task t = job.obtainNewMapTask(taskTracker, tts);
          if (t != null) {
            return t;
          }

          //
          // Beyond the highest-priority task, reserve a little
          // room for failures and speculative executions; don't
          // schedule tasks to the hilt.
          //
          totalNeededMaps += job.desiredMaps();
          double padding = 0;
          if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
            padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION);
          }
          if (totalNeededMaps + padding >= totalCapacity) {
            break;
          }
        }
      }

      //
      // Same thing, but for reduce tasks
      //
      if ((numReduces < maxCurrentTasks) && (numReduces <= (avgReduces + TASK_ALLOC_EPSILON))) {

        int totalNeededReduces = 0;
        for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
          JobInProgress job = (JobInProgress) it.next();
          if (job.getStatus().getRunState() != JobStatus.RUNNING) {
            continue;
          }

          Task t = job.obtainNewReduceTask(taskTracker, tts);
          if (t != null) {
            return t;
          }

          //
          // Beyond the highest-priority task, reserve a little
          // room for failures and speculative executions; don't
          // schedule tasks to the hilt.
          //
          totalNeededReduces += job.desiredReduces();
          double padding = 0;
          if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
            padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION);
          }
          if (totalNeededReduces + padding >= totalCapacity) {
            break;
          }
        }
      }
    }
    return null;
  }
Exemplo n.º 2
0
 public synchronized ClusterStatus getClusterStatus() {
   synchronized (taskTrackers) {
     return new ClusterStatus(taskTrackers.size(), totalMaps, totalReduces, maxCurrentTasks);
   }
 }
Exemplo n.º 3
0
  /**
   * Process an HTML get or post.
   *
   * @exception ServletException From inherited class.
   * @exception IOException From inherited class.
   */
  public void scanOutXML(
      PrintWriter out,
      String strDirectory,
      String strFilename,
      String[] strPlus,
      String[] strMinus,
      boolean bExcludeParams,
      boolean bAnalyzeParams)
      throws IOException {
    File dir = new File(strDirectory + '/' + strFilename);
    if (dir.isDirectory()) return;

    try {
      FileReader is = new FileReader(strDirectory + '/' + strFilename);
      BufferedReader r = new BufferedReader(is);
      String string = null;
      Hashtable ht = new Hashtable();
      Set setstrExtensions = new HashSet();
      int iCount = 0;
      int iBytes = 0;
      while ((string = r.readLine()) != null) {
        StringTokenizer st = new StringTokenizer(string, " \"", false);
        Data data = new Data();
        int iTokenCount = 0;
        while (st.hasMoreTokens()) {
          iTokenCount++;
          string = st.nextToken();
          if (iTokenCount == IP) data.m_IP = string;
          if (iTokenCount == URL) {
            if (bExcludeParams)
              if (string.indexOf('?') != -1) string = string.substring(0, string.indexOf('?'));
            if (bAnalyzeParams)
              if (string.indexOf('?') != -1) string = string.substring(string.indexOf('?') + 1);
            data.m_URL = string;
          }
          if (iTokenCount == PROTOCOL)
            if (!string.startsWith("HTTP")) {
              data.m_URL += " " + string;
              iTokenCount--;
            }
          if (iTokenCount == BYTES) data.m_iBytes = Integer.parseInt(string);
        }
        if (!this.filterURL(data.m_URL, strPlus, strMinus, setstrExtensions)) continue;
        iCount++;
        iBytes += data.m_iBytes;
        if (ht.get(data.m_URL) == null) ht.put(data.m_URL, data);
        else {
          int iThisBytes = data.m_iBytes;
          data = (Data) ht.get(data.m_URL);
          data.m_iCount++;
          data.m_iBytes += iThisBytes;
        }
      }
      Comparator comparator = new Test();
      TreeMap tm = new TreeMap(comparator);
      Iterator iterator = ht.values().iterator();
      while (iterator.hasNext()) {
        Data data = (Data) iterator.next();
        tm.put(new Integer(data.m_iCount), data);
      }
      out.println("<file>");
      this.printXML(out, "directory", strDirectory);
      this.printXML(out, "name", strFilename);
      iterator = tm.values().iterator();
      while (iterator.hasNext()) {
        out.println("<data>");
        Data data = (Data) iterator.next();
        this.printXML(out, "url", data.m_URL);
        this.printXML(out, "count", Integer.toString(data.m_iCount));
        out.println("</data>");
      }
      this.printXML(out, "hits", Integer.toString(iCount));
      this.printXML(out, "bytes", Integer.toString(iBytes));
      this.printXML(out, "unique", Integer.toString(tm.size()));

      iterator = setstrExtensions.iterator();
      out.println("<extensions>");
      while (iterator.hasNext()) {
        this.printXML(out, "extension", (String) iterator.next());
      }
      out.println("</extensions>");

      out.println("</file>");
    } catch (FileNotFoundException ex) {
      ex.printStackTrace();
    } catch (IOException ex) {
      ex.printStackTrace();
    }
  }