/**
* Checks if the map-reduce job has completed.
*
* @return true if the job completed, false otherwise.
* @throws IOException
*/
public boolean checkComplete() throws IOException {
JobID jobID = runningJob.getID();
if (runningJob.isComplete()) {
// delete job directory
final String jobdir = jobconf.get(JOB_DIR_LABEL);
if (jobdir != null) {
final Path jobpath = new Path(jobdir);
jobpath.getFileSystem(jobconf).delete(jobpath, true);
}
if (runningJob.isSuccessful()) {
LOG.info("Job Complete(Succeeded): " + jobID);
} else {
LOG.info("Job Complete(Failed): " + jobID);
}
raidPolicyPathPairList.clear();
Counters ctrs = runningJob.getCounters();
if (ctrs != null) {
RaidNodeMetrics metrics = RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID);
if (ctrs.findCounter(Counter.FILES_FAILED) != null) {
long filesFailed = ctrs.findCounter(Counter.FILES_FAILED).getValue();
metrics.raidFailures.inc(filesFailed);
}
long slotSeconds =
ctrs.findCounter(JobInProgress.Counter.SLOTS_MILLIS_MAPS).getValue() / 1000;
metrics.raidSlotSeconds.inc(slotSeconds);
}
return true;
} else {
String report =
(" job "
+ jobID
+ " map "
+ StringUtils.formatPercent(runningJob.mapProgress(), 0)
+ " reduce "
+ StringUtils.formatPercent(runningJob.reduceProgress(), 0));
if (!report.equals(lastReport)) {
LOG.info(report);
lastReport = report;
}
TaskCompletionEvent[] events = runningJob.getTaskCompletionEvents(jobEventCounter);
jobEventCounter += events.length;
for (TaskCompletionEvent event : events) {
if (event.getTaskStatus() == TaskCompletionEvent.Status.FAILED) {
LOG.info(" Job " + jobID + " " + event.toString());
}
}
return false;
}
}
public void logStats() {
long milliseconds = this.timeSinceLastAccess.get() / 1000000;
LOG.info(
"For Slab of size "
+ this.blockSize
+ ": "
+ this.getOccupiedSize() / this.blockSize
+ " occupied, out of a capacity of "
+ this.numBlocks
+ " blocks. HeapSize is "
+ StringUtils.humanReadableInt(this.heapSize())
+ " bytes."
+ ", "
+ "churnTime="
+ StringUtils.formatTime(milliseconds));
LOG.info(
"Slab Stats: "
+ "accesses="
+ stats.getRequestCount()
+ ", "
+ "hits="
+ stats.getHitCount()
+ ", "
+ "hitRatio="
+ (stats.getHitCount() == 0
? "0"
: (StringUtils.formatPercent(stats.getHitRatio(), 2) + "%, "))
+ "cachingAccesses="
+ stats.getRequestCachingCount()
+ ", "
+ "cachingHits="
+ stats.getHitCachingCount()
+ ", "
+ "cachingHitsRatio="
+ (stats.getHitCachingCount() == 0
? "0"
: (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + "%, "))
+ "evictions="
+ stats.getEvictionCount()
+ ", "
+ "evicted="
+ stats.getEvictedCount()
+ ", "
+ "evictedPerRun="
+ stats.evictedPerEviction());
}
public JobInfo(Job job, Boolean hasAccess) {
this.id = MRApps.toString(job.getID());
JobReport report = job.getReport();
this.startTime = report.getStartTime();
this.finishTime = report.getFinishTime();
this.elapsedTime = Times.elapsed(this.startTime, this.finishTime);
if (this.elapsedTime == -1) {
this.elapsedTime = 0;
}
this.name = job.getName().toString();
this.user = job.getUserName();
this.queue = job.getQueueName();
this.state = job.getState();
this.mapsTotal = job.getTotalMaps();
this.mapsCompleted = job.getCompletedMaps();
this.mapProgress = report.getMapProgress() * 100;
this.mapProgressPercent = StringUtils.formatPercent(report.getMapProgress(), 2);
this.reducesTotal = job.getTotalReduces();
this.reducesCompleted = job.getCompletedReduces();
this.reduceProgress = report.getReduceProgress() * 100;
this.reduceProgressPercent = StringUtils.formatPercent(report.getReduceProgress(), 2);
this.acls = new ArrayList<ConfEntryInfo>();
if (hasAccess) {
this.diagnostics = "";
countTasksAndAttempts(job);
this.uberized = job.isUber();
List<String> diagnostics = job.getDiagnostics();
if (diagnostics != null && !diagnostics.isEmpty()) {
StringBuffer b = new StringBuffer();
for (String diag : diagnostics) {
b.append(diag);
}
this.diagnostics = b.toString();
}
Map<JobACL, AccessControlList> allacls = job.getJobACLs();
if (allacls != null) {
for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) {
this.acls.add(
new ConfEntryInfo(entry.getKey().getAclName(), entry.getValue().getAclString()));
}
}
}
}