/** * Checks if the map-reduce job has completed. * * @return true if the job completed, false otherwise. * @throws IOException */ public boolean checkComplete() throws IOException { JobID jobID = runningJob.getID(); if (runningJob.isComplete()) { // delete job directory final String jobdir = jobconf.get(JOB_DIR_LABEL); if (jobdir != null) { final Path jobpath = new Path(jobdir); jobpath.getFileSystem(jobconf).delete(jobpath, true); } if (runningJob.isSuccessful()) { LOG.info("Job Complete(Succeeded): " + jobID); } else { LOG.info("Job Complete(Failed): " + jobID); } raidPolicyPathPairList.clear(); Counters ctrs = runningJob.getCounters(); if (ctrs != null) { RaidNodeMetrics metrics = RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID); if (ctrs.findCounter(Counter.FILES_FAILED) != null) { long filesFailed = ctrs.findCounter(Counter.FILES_FAILED).getValue(); metrics.raidFailures.inc(filesFailed); } long slotSeconds = ctrs.findCounter(JobInProgress.Counter.SLOTS_MILLIS_MAPS).getValue() / 1000; metrics.raidSlotSeconds.inc(slotSeconds); } return true; } else { String report = (" job " + jobID + " map " + StringUtils.formatPercent(runningJob.mapProgress(), 0) + " reduce " + StringUtils.formatPercent(runningJob.reduceProgress(), 0)); if (!report.equals(lastReport)) { LOG.info(report); lastReport = report; } TaskCompletionEvent[] events = runningJob.getTaskCompletionEvents(jobEventCounter); jobEventCounter += events.length; for (TaskCompletionEvent event : events) { if (event.getTaskStatus() == TaskCompletionEvent.Status.FAILED) { LOG.info(" Job " + jobID + " " + event.toString()); } } return false; } }
public void logStats() { long milliseconds = this.timeSinceLastAccess.get() / 1000000; LOG.info( "For Slab of size " + this.blockSize + ": " + this.getOccupiedSize() / this.blockSize + " occupied, out of a capacity of " + this.numBlocks + " blocks. HeapSize is " + StringUtils.humanReadableInt(this.heapSize()) + " bytes." + ", " + "churnTime=" + StringUtils.formatTime(milliseconds)); LOG.info( "Slab Stats: " + "accesses=" + stats.getRequestCount() + ", " + "hits=" + stats.getHitCount() + ", " + "hitRatio=" + (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2) + "%, ")) + "cachingAccesses=" + stats.getRequestCachingCount() + ", " + "cachingHits=" + stats.getHitCachingCount() + ", " + "cachingHitsRatio=" + (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + "%, ")) + "evictions=" + stats.getEvictionCount() + ", " + "evicted=" + stats.getEvictedCount() + ", " + "evictedPerRun=" + stats.evictedPerEviction()); }
public JobInfo(Job job, Boolean hasAccess) { this.id = MRApps.toString(job.getID()); JobReport report = job.getReport(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.elapsedTime = Times.elapsed(this.startTime, this.finishTime); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.name = job.getName().toString(); this.user = job.getUserName(); this.queue = job.getQueueName(); this.state = job.getState(); this.mapsTotal = job.getTotalMaps(); this.mapsCompleted = job.getCompletedMaps(); this.mapProgress = report.getMapProgress() * 100; this.mapProgressPercent = StringUtils.formatPercent(report.getMapProgress(), 2); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.reduceProgress = report.getReduceProgress() * 100; this.reduceProgressPercent = StringUtils.formatPercent(report.getReduceProgress(), 2); this.acls = new ArrayList<ConfEntryInfo>(); if (hasAccess) { this.diagnostics = ""; countTasksAndAttempts(job); this.uberized = job.isUber(); List<String> diagnostics = job.getDiagnostics(); if (diagnostics != null && !diagnostics.isEmpty()) { StringBuffer b = new StringBuffer(); for (String diag : diagnostics) { b.append(diag); } this.diagnostics = b.toString(); } Map<JobACL, AccessControlList> allacls = job.getJobACLs(); if (allacls != null) { for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) { this.acls.add( new ConfEntryInfo(entry.getKey().getAclName(), entry.getValue().getAclString())); } } } }