コード例 #1
0
 public void processEventForJobSummary(HistoryEvent event, JobSummary summary, JobId jobId) {
   // context.getJob could be used for some of this info as well.
   switch (event.getEventType()) {
     case JOB_SUBMITTED:
       JobSubmittedEvent jse = (JobSubmittedEvent) event;
       summary.setUser(jse.getUserName());
       summary.setQueue(jse.getJobQueueName());
       summary.setJobSubmitTime(jse.getSubmitTime());
       summary.setJobName(jse.getJobName());
       break;
     case NORMALIZED_RESOURCE:
       NormalizedResourceEvent normalizedResourceEvent = (NormalizedResourceEvent) event;
       if (normalizedResourceEvent.getTaskType() == TaskType.MAP) {
         summary.setResourcesPerMap(normalizedResourceEvent.getMemory());
       } else if (normalizedResourceEvent.getTaskType() == TaskType.REDUCE) {
         summary.setResourcesPerReduce(normalizedResourceEvent.getMemory());
       }
       break;
     case JOB_INITED:
       JobInitedEvent jie = (JobInitedEvent) event;
       summary.setJobLaunchTime(jie.getLaunchTime());
       break;
     case MAP_ATTEMPT_STARTED:
       TaskAttemptStartedEvent mtase = (TaskAttemptStartedEvent) event;
       if (summary.getFirstMapTaskLaunchTime() == 0)
         summary.setFirstMapTaskLaunchTime(mtase.getStartTime());
       break;
     case REDUCE_ATTEMPT_STARTED:
       TaskAttemptStartedEvent rtase = (TaskAttemptStartedEvent) event;
       if (summary.getFirstReduceTaskLaunchTime() == 0)
         summary.setFirstReduceTaskLaunchTime(rtase.getStartTime());
       break;
     case JOB_FINISHED:
       JobFinishedEvent jfe = (JobFinishedEvent) event;
       summary.setJobFinishTime(jfe.getFinishTime());
       summary.setNumFinishedMaps(jfe.getFinishedMaps());
       summary.setNumFailedMaps(jfe.getFailedMaps());
       summary.setNumFinishedReduces(jfe.getFinishedReduces());
       summary.setNumFailedReduces(jfe.getFailedReduces());
       if (summary.getJobStatus() == null)
         summary.setJobStatus(org.apache.hadoop.mapreduce.JobStatus.State.SUCCEEDED.toString());
       // TODO JOB_FINISHED does not have state. Effectively job history does not
       // have state about the finished job.
       setSummarySlotSeconds(summary, jfe.getTotalCounters());
       break;
     case JOB_FAILED:
     case JOB_KILLED:
       JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event;
       summary.setJobStatus(juce.getStatus());
       summary.setNumFinishedMaps(context.getJob(jobId).getTotalMaps());
       summary.setNumFinishedReduces(context.getJob(jobId).getTotalReduces());
       summary.setJobFinishTime(juce.getFinishTime());
       setSummarySlotSeconds(summary, context.getJob(jobId).getAllCounters());
       break;
   }
 }
コード例 #2
0
  protected void handleEvent(JobHistoryEvent event) {
    synchronized (lock) {

      // If this is JobSubmitted Event, setup the writer
      if (event.getHistoryEvent().getEventType() == EventType.AM_STARTED) {
        try {
          setupEventWriter(event.getJobID());
        } catch (IOException ioe) {
          LOG.error("Error JobHistoryEventHandler in handleEvent: " + event, ioe);
          throw new YarnException(ioe);
        }
      }

      // For all events
      // (1) Write it out
      // (2) Process it for JobSummary
      MetaInfo mi = fileMap.get(event.getJobID());
      try {
        HistoryEvent historyEvent = event.getHistoryEvent();
        if (!(historyEvent instanceof NormalizedResourceEvent)) {
          mi.writeEvent(historyEvent);
        }
        processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID());
        if (LOG.isDebugEnabled()) {
          LOG.debug("In HistoryEventHandler " + event.getHistoryEvent().getEventType());
        }
      } catch (IOException e) {
        LOG.error("Error writing History Event: " + event.getHistoryEvent(), e);
        throw new YarnException(e);
      }

      if (event.getHistoryEvent().getEventType() == EventType.JOB_SUBMITTED) {
        JobSubmittedEvent jobSubmittedEvent = (JobSubmittedEvent) event.getHistoryEvent();
        mi.getJobIndexInfo().setSubmitTime(jobSubmittedEvent.getSubmitTime());
        mi.getJobIndexInfo().setQueueName(jobSubmittedEvent.getJobQueueName());
      }

      // If this is JobFinishedEvent, close the writer and setup the job-index
      if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) {
        try {
          JobFinishedEvent jFinishedEvent = (JobFinishedEvent) event.getHistoryEvent();
          mi.getJobIndexInfo().setFinishTime(jFinishedEvent.getFinishTime());
          mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getFinishedMaps());
          mi.getJobIndexInfo().setNumReduces(jFinishedEvent.getFinishedReduces());
          mi.getJobIndexInfo().setJobStatus(JobState.SUCCEEDED.toString());
          closeEventWriter(event.getJobID());
        } catch (IOException e) {
          throw new YarnException(e);
        }
      }

      if (event.getHistoryEvent().getEventType() == EventType.JOB_FAILED
          || event.getHistoryEvent().getEventType() == EventType.JOB_KILLED) {
        try {
          JobUnsuccessfulCompletionEvent jucEvent =
              (JobUnsuccessfulCompletionEvent) event.getHistoryEvent();
          mi.getJobIndexInfo().setFinishTime(jucEvent.getFinishTime());
          mi.getJobIndexInfo().setNumMaps(jucEvent.getFinishedMaps());
          mi.getJobIndexInfo().setNumReduces(jucEvent.getFinishedReduces());
          mi.getJobIndexInfo().setJobStatus(jucEvent.getStatus());
          closeEventWriter(event.getJobID());
        } catch (IOException e) {
          throw new YarnException(e);
        }
      }
    }
  }