コード例 #1
0
 @Override
 public void reportTaskTrackerUtilization(
     TaskTrackerUtilization ttUtil, LocalJobUtilization[] localJobUtil) throws IOException {
   UtilizationReport utilReport = new UtilizationReport();
   utilReport.setValues(ttUtil, localJobUtil);
   taskTrackerReports.put(ttUtil.getHostName(), utilReport);
 }
コード例 #2
0
  /** Aggregate the TaskTracker reports */
  protected void aggregateReports() {
    clusterUtil.clear();
    for (String jobId : allJobUtil.keySet()) {
      JobUtilization jobUtil = allJobUtil.get(jobId);
      jobUtil.clear();
      jobUtil.setIsRunning(false);
    }

    for (UtilizationReport report : taskTrackerReports.values()) {
      if (report.isExpired()) {
        continue; // ignore the report if it is older than the timeLimit
      }
      LocalJobUtilization[] localJobUtils = report.getJobUtilization();
      TaskTrackerUtilization ttUtil = report.getTaskTrackerUtilization();

      // Aggregate cluster information
      double cpuUsageGHz = clusterUtil.getCpuUsageGHz();
      double cpuTotalGHz = clusterUtil.getCpuTotalGHz();
      double memUsageGB = clusterUtil.getMemUsageGB();
      double memTotalGB = clusterUtil.getMemTotalGB();
      int numCpu = clusterUtil.getNumCpu();
      int numTaskTrackers = clusterUtil.getNumTaskTrackers();

      cpuUsageGHz += ttUtil.getCpuUsageGHz();
      cpuTotalGHz += ttUtil.getCpuTotalGHz();
      memUsageGB += ttUtil.getMemUsageGB();
      memTotalGB += ttUtil.getMemTotalGB();
      numCpu += ttUtil.getNumCpu();
      numTaskTrackers += 1;

      clusterUtil.setCpuUsageGHz(cpuUsageGHz);
      clusterUtil.setCpuTotalGHz(cpuTotalGHz);
      clusterUtil.setMemUsageGB(memUsageGB);
      clusterUtil.setMemTotalGB(memTotalGB);
      clusterUtil.setNumCpu(numCpu);
      clusterUtil.setNumTaskTrackers(numTaskTrackers);

      // Aggregate Job based information
      if (localJobUtils == null) {
        continue;
      }

      for (LocalJobUtilization localJobUtil : localJobUtils) {
        String jobId = localJobUtil.getJobId();
        if (jobId == null) {
          continue;
        }

        if (!allJobUtil.containsKey(jobId)) {
          JobUtilization jobUtil = new JobUtilization();
          jobUtil.setJobId(jobId);
          allJobUtil.put(jobId, jobUtil);
        }
        JobUtilization jobUtil = allJobUtil.get(jobId);
        jobUtil.setIsRunning(true);
        double maxCpu = jobUtil.getCpuMaxPercentageOnBox();
        double cpu = jobUtil.getCpuPercentageOnCluster();
        double cpuGigaCycles = jobUtil.getCpuGigaCycles();
        double maxMem = jobUtil.getMemMaxPercentageOnBox();
        double mem = jobUtil.getMemPercentageOnCluster();

        cpu += localJobUtil.getCpuUsageGHz(); // will be normalized later
        mem += localJobUtil.getMemUsageGB(); // will be normalized later
        cpuGigaCycles += localJobUtil.getCpuUsageGHz() * (double) aggregatePeriod * 1e6D / 1e9D;
        // GHz * ms = 1e6. To convert to Giga, divide by 1e9

        double localCpuPercentage = localJobUtil.getCpuUsageGHz() / ttUtil.getCpuTotalGHz() * 100;
        double localMemPercentage = localJobUtil.getMemUsageGB() / ttUtil.getMemTotalGB() * 100;

        if (maxCpu < localCpuPercentage) {
          maxCpu = localCpuPercentage;
        }
        if (maxMem < localMemPercentage) {
          maxMem = localMemPercentage;
        }
        jobUtil.setCpuMaxPercentageOnBox(maxCpu);
        jobUtil.setCpuGigaCycles(cpuGigaCycles);
        jobUtil.setCpuPercentageOnCluster(cpu); // will be normalized later
        jobUtil.setMemMaxPercentageOnBox(maxMem); // will be normalized later
        jobUtil.setMemPercentageOnCluster(mem);
        if (maxMem > jobUtil.getMemMaxPercentageOnBoxAllTime()) {
          jobUtil.setMemMaxPercentageOnBoxAllTime(maxMem);
        }
      }
    }

    // Normalization and clean up finished jobs
    for (Iterator<String> it = allJobUtil.keySet().iterator(); it.hasNext(); ) {
      String jobId = it.next();
      JobUtilization jobUtil = allJobUtil.get(jobId);
      if (!jobUtil.getIsRunning()) {
        long stoppedTime = jobUtil.getStoppedTime();
        stoppedTime += aggregatePeriod;
        jobUtil.setStoppedTime(stoppedTime);
        jobUtil.setIsRunning(false);

        if (stoppedTime > stopTimeLimit) {
          // These are finished jobs
          // We may store the information of finished jobs to some place
          double cpuTime = jobUtil.getCpuCumulatedUsageTime() / 1000D;
          double memTime = jobUtil.getMemCumulatedUsageTime() / 1000D;
          double peakMem = jobUtil.getMemMaxPercentageOnBoxAllTime();
          double cpuGigaCycles = jobUtil.getCpuGigaCycles();
          it.remove();
          LOG.info(
              String.format(
                  "Job done: [JobID,CPU(sec),Mem(sec),Peak Mem(%%),CPU gigacycles]"
                      + " = [%s,%f,%f,%f,%f]",
                  jobId, cpuTime, memTime, peakMem, cpuGigaCycles));
        }
        continue;
      }

      long runningTime = jobUtil.getRunningTime();
      runningTime += aggregatePeriod; // millisecond to second
      jobUtil.setRunningTime(runningTime);
      jobUtil.setStoppedTime(0);
      jobUtil.setIsRunning(true);

      int numJobs = clusterUtil.getNumRunningJobs();
      numJobs += 1;
      clusterUtil.setNumRunningJobs(numJobs);

      double cpu = jobUtil.getCpuPercentageOnCluster();
      double mem = jobUtil.getMemPercentageOnCluster();
      double cpuTime = jobUtil.getCpuCumulatedUsageTime();
      double memTime = jobUtil.getMemCumulatedUsageTime();
      cpu = cpu / clusterUtil.getCpuTotalGHz() * 100;
      mem = mem / clusterUtil.getMemTotalGB() * 100;
      cpuTime += cpu / 100 * aggregatePeriod; // in milliseconds
      memTime += mem / 100 * aggregatePeriod; // in milliseconds
      jobUtil.setCpuPercentageOnCluster(cpu);
      jobUtil.setMemPercentageOnCluster(mem);
      jobUtil.setCpuCumulatedUsageTime(cpuTime);
      jobUtil.setMemCumulatedUsageTime(memTime);
    }
  }