@Override
public void reportTaskTrackerUtilization(
TaskTrackerUtilization ttUtil, LocalJobUtilization[] localJobUtil) throws IOException {
UtilizationReport utilReport = new UtilizationReport();
utilReport.setValues(ttUtil, localJobUtil);
taskTrackerReports.put(ttUtil.getHostName(), utilReport);
}
/** Aggregate the TaskTracker reports */
protected void aggregateReports() {
clusterUtil.clear();
for (String jobId : allJobUtil.keySet()) {
JobUtilization jobUtil = allJobUtil.get(jobId);
jobUtil.clear();
jobUtil.setIsRunning(false);
}
for (UtilizationReport report : taskTrackerReports.values()) {
if (report.isExpired()) {
continue; // ignore the report if it is older than the timeLimit
}
LocalJobUtilization[] localJobUtils = report.getJobUtilization();
TaskTrackerUtilization ttUtil = report.getTaskTrackerUtilization();
// Aggregate cluster information
double cpuUsageGHz = clusterUtil.getCpuUsageGHz();
double cpuTotalGHz = clusterUtil.getCpuTotalGHz();
double memUsageGB = clusterUtil.getMemUsageGB();
double memTotalGB = clusterUtil.getMemTotalGB();
int numCpu = clusterUtil.getNumCpu();
int numTaskTrackers = clusterUtil.getNumTaskTrackers();
cpuUsageGHz += ttUtil.getCpuUsageGHz();
cpuTotalGHz += ttUtil.getCpuTotalGHz();
memUsageGB += ttUtil.getMemUsageGB();
memTotalGB += ttUtil.getMemTotalGB();
numCpu += ttUtil.getNumCpu();
numTaskTrackers += 1;
clusterUtil.setCpuUsageGHz(cpuUsageGHz);
clusterUtil.setCpuTotalGHz(cpuTotalGHz);
clusterUtil.setMemUsageGB(memUsageGB);
clusterUtil.setMemTotalGB(memTotalGB);
clusterUtil.setNumCpu(numCpu);
clusterUtil.setNumTaskTrackers(numTaskTrackers);
// Aggregate Job based information
if (localJobUtils == null) {
continue;
}
for (LocalJobUtilization localJobUtil : localJobUtils) {
String jobId = localJobUtil.getJobId();
if (jobId == null) {
continue;
}
if (!allJobUtil.containsKey(jobId)) {
JobUtilization jobUtil = new JobUtilization();
jobUtil.setJobId(jobId);
allJobUtil.put(jobId, jobUtil);
}
JobUtilization jobUtil = allJobUtil.get(jobId);
jobUtil.setIsRunning(true);
double maxCpu = jobUtil.getCpuMaxPercentageOnBox();
double cpu = jobUtil.getCpuPercentageOnCluster();
double cpuGigaCycles = jobUtil.getCpuGigaCycles();
double maxMem = jobUtil.getMemMaxPercentageOnBox();
double mem = jobUtil.getMemPercentageOnCluster();
cpu += localJobUtil.getCpuUsageGHz(); // will be normalized later
mem += localJobUtil.getMemUsageGB(); // will be normalized later
cpuGigaCycles += localJobUtil.getCpuUsageGHz() * (double) aggregatePeriod * 1e6D / 1e9D;
// GHz * ms = 1e6. To convert to Giga, divide by 1e9
double localCpuPercentage = localJobUtil.getCpuUsageGHz() / ttUtil.getCpuTotalGHz() * 100;
double localMemPercentage = localJobUtil.getMemUsageGB() / ttUtil.getMemTotalGB() * 100;
if (maxCpu < localCpuPercentage) {
maxCpu = localCpuPercentage;
}
if (maxMem < localMemPercentage) {
maxMem = localMemPercentage;
}
jobUtil.setCpuMaxPercentageOnBox(maxCpu);
jobUtil.setCpuGigaCycles(cpuGigaCycles);
jobUtil.setCpuPercentageOnCluster(cpu); // will be normalized later
jobUtil.setMemMaxPercentageOnBox(maxMem); // will be normalized later
jobUtil.setMemPercentageOnCluster(mem);
if (maxMem > jobUtil.getMemMaxPercentageOnBoxAllTime()) {
jobUtil.setMemMaxPercentageOnBoxAllTime(maxMem);
}
}
}
// Normalization and clean up finished jobs
for (Iterator<String> it = allJobUtil.keySet().iterator(); it.hasNext(); ) {
String jobId = it.next();
JobUtilization jobUtil = allJobUtil.get(jobId);
if (!jobUtil.getIsRunning()) {
long stoppedTime = jobUtil.getStoppedTime();
stoppedTime += aggregatePeriod;
jobUtil.setStoppedTime(stoppedTime);
jobUtil.setIsRunning(false);
if (stoppedTime > stopTimeLimit) {
// These are finished jobs
// We may store the information of finished jobs to some place
double cpuTime = jobUtil.getCpuCumulatedUsageTime() / 1000D;
double memTime = jobUtil.getMemCumulatedUsageTime() / 1000D;
double peakMem = jobUtil.getMemMaxPercentageOnBoxAllTime();
double cpuGigaCycles = jobUtil.getCpuGigaCycles();
it.remove();
LOG.info(
String.format(
"Job done: [JobID,CPU(sec),Mem(sec),Peak Mem(%%),CPU gigacycles]"
+ " = [%s,%f,%f,%f,%f]",
jobId, cpuTime, memTime, peakMem, cpuGigaCycles));
}
continue;
}
long runningTime = jobUtil.getRunningTime();
runningTime += aggregatePeriod; // millisecond to second
jobUtil.setRunningTime(runningTime);
jobUtil.setStoppedTime(0);
jobUtil.setIsRunning(true);
int numJobs = clusterUtil.getNumRunningJobs();
numJobs += 1;
clusterUtil.setNumRunningJobs(numJobs);
double cpu = jobUtil.getCpuPercentageOnCluster();
double mem = jobUtil.getMemPercentageOnCluster();
double cpuTime = jobUtil.getCpuCumulatedUsageTime();
double memTime = jobUtil.getMemCumulatedUsageTime();
cpu = cpu / clusterUtil.getCpuTotalGHz() * 100;
mem = mem / clusterUtil.getMemTotalGB() * 100;
cpuTime += cpu / 100 * aggregatePeriod; // in milliseconds
memTime += mem / 100 * aggregatePeriod; // in milliseconds
jobUtil.setCpuPercentageOnCluster(cpu);
jobUtil.setMemPercentageOnCluster(mem);
jobUtil.setCpuCumulatedUsageTime(cpuTime);
jobUtil.setMemCumulatedUsageTime(memTime);
}
}