private boolean isJobCompletionEvent(HistoryEvent historyEvent) {
if (EnumSet.of(EventType.JOB_FINISHED, EventType.JOB_FAILED, EventType.JOB_KILLED)
.contains(historyEvent.getEventType())) {
return true;
}
return false;
}
public void processEventForJobSummary(HistoryEvent event, JobSummary summary, JobId jobId) {
// context.getJob could be used for some of this info as well.
switch (event.getEventType()) {
case JOB_SUBMITTED:
JobSubmittedEvent jse = (JobSubmittedEvent) event;
summary.setUser(jse.getUserName());
summary.setQueue(jse.getJobQueueName());
summary.setJobSubmitTime(jse.getSubmitTime());
summary.setJobName(jse.getJobName());
break;
case NORMALIZED_RESOURCE:
NormalizedResourceEvent normalizedResourceEvent = (NormalizedResourceEvent) event;
if (normalizedResourceEvent.getTaskType() == TaskType.MAP) {
summary.setResourcesPerMap(normalizedResourceEvent.getMemory());
} else if (normalizedResourceEvent.getTaskType() == TaskType.REDUCE) {
summary.setResourcesPerReduce(normalizedResourceEvent.getMemory());
}
break;
case JOB_INITED:
JobInitedEvent jie = (JobInitedEvent) event;
summary.setJobLaunchTime(jie.getLaunchTime());
break;
case MAP_ATTEMPT_STARTED:
TaskAttemptStartedEvent mtase = (TaskAttemptStartedEvent) event;
if (summary.getFirstMapTaskLaunchTime() == 0)
summary.setFirstMapTaskLaunchTime(mtase.getStartTime());
break;
case REDUCE_ATTEMPT_STARTED:
TaskAttemptStartedEvent rtase = (TaskAttemptStartedEvent) event;
if (summary.getFirstReduceTaskLaunchTime() == 0)
summary.setFirstReduceTaskLaunchTime(rtase.getStartTime());
break;
case JOB_FINISHED:
JobFinishedEvent jfe = (JobFinishedEvent) event;
summary.setJobFinishTime(jfe.getFinishTime());
summary.setNumFinishedMaps(jfe.getFinishedMaps());
summary.setNumFailedMaps(jfe.getFailedMaps());
summary.setNumFinishedReduces(jfe.getFinishedReduces());
summary.setNumFailedReduces(jfe.getFailedReduces());
if (summary.getJobStatus() == null)
summary.setJobStatus(org.apache.hadoop.mapreduce.JobStatus.State.SUCCEEDED.toString());
// TODO JOB_FINISHED does not have state. Effectively job history does not
// have state about the finished job.
setSummarySlotSeconds(summary, jfe.getTotalCounters());
break;
case JOB_FAILED:
case JOB_KILLED:
JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event;
summary.setJobStatus(juce.getStatus());
summary.setNumFinishedMaps(context.getJob(jobId).getTotalMaps());
summary.setNumFinishedReduces(context.getJob(jobId).getTotalReduces());
summary.setJobFinishTime(juce.getFinishTime());
setSummarySlotSeconds(summary, context.getJob(jobId).getAllCounters());
break;
}
}
void processEventForFlush(HistoryEvent historyEvent) throws IOException {
if (EnumSet.of(
EventType.MAP_ATTEMPT_FINISHED,
EventType.MAP_ATTEMPT_FAILED,
EventType.MAP_ATTEMPT_KILLED,
EventType.REDUCE_ATTEMPT_FINISHED,
EventType.REDUCE_ATTEMPT_FAILED,
EventType.REDUCE_ATTEMPT_KILLED,
EventType.TASK_FINISHED,
EventType.TASK_FAILED,
EventType.JOB_FINISHED,
EventType.JOB_FAILED,
EventType.JOB_KILLED)
.contains(historyEvent.getEventType())) {
numUnflushedCompletionEvents++;
if (!isTimerActive) {
resetFlushTimer();
if (!isTimerShutDown) {
flushTimerTask = new FlushTimerTask(this);
flushTimer.schedule(flushTimerTask, flushTimeout);
}
}
}
}
/**
* Get the next event from the stream
*
* @return the next event
* @throws IOException
*/
@SuppressWarnings("unchecked")
public HistoryEvent getNextEvent() throws IOException {
Event wrapper;
try {
wrapper = (Event) reader.read(null, decoder);
} catch (EOFException e) { // at EOF
return null;
}
HistoryEvent result;
switch (wrapper.type) {
case JOB_SUBMITTED:
result = new JobSubmittedEvent();
break;
case JOB_INITED:
result = new JobInitedEvent();
break;
case JOB_FINISHED:
result = new JobFinishedEvent();
break;
case JOB_PRIORITY_CHANGED:
result = new JobPriorityChangeEvent();
break;
case JOB_STATUS_CHANGED:
result = new JobStatusChangedEvent();
break;
case JOB_FAILED:
result = new JobUnsuccessfulCompletionEvent();
break;
case JOB_KILLED:
result = new JobUnsuccessfulCompletionEvent();
break;
case JOB_INFO_CHANGED:
result = new JobInfoChangeEvent();
break;
case TASK_STARTED:
result = new TaskStartedEvent();
break;
case TASK_FINISHED:
result = new TaskFinishedEvent();
break;
case TASK_FAILED:
result = new TaskFailedEvent();
break;
case TASK_UPDATED:
result = new TaskUpdatedEvent();
break;
case MAP_ATTEMPT_STARTED:
result = new TaskAttemptStartedEvent();
break;
case MAP_ATTEMPT_FINISHED:
result = new MapAttemptFinishedEvent();
break;
case MAP_ATTEMPT_FAILED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case MAP_ATTEMPT_KILLED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case REDUCE_ATTEMPT_STARTED:
result = new TaskAttemptStartedEvent();
break;
case REDUCE_ATTEMPT_FINISHED:
result = new ReduceAttemptFinishedEvent();
break;
case REDUCE_ATTEMPT_FAILED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case REDUCE_ATTEMPT_KILLED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case SETUP_ATTEMPT_STARTED:
result = new TaskAttemptStartedEvent();
break;
case SETUP_ATTEMPT_FINISHED:
result = new TaskAttemptFinishedEvent();
break;
case SETUP_ATTEMPT_FAILED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case SETUP_ATTEMPT_KILLED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case CLEANUP_ATTEMPT_STARTED:
result = new TaskAttemptStartedEvent();
break;
case CLEANUP_ATTEMPT_FINISHED:
result = new TaskAttemptFinishedEvent();
break;
case CLEANUP_ATTEMPT_FAILED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case CLEANUP_ATTEMPT_KILLED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case AM_STARTED:
result = new AMStartedEvent();
break;
default:
throw new RuntimeException("unexpected event type: " + wrapper.type);
}
result.setDatum(wrapper.event);
return result;
}
/** Test if {@link CurrentJHParser} can read events from current JH files. */
@Test
public void testCurrentJHParser() throws Exception {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
final Path tempDir = new Path(rootTempDir, "TestCurrentJHParser");
lfs.delete(tempDir, true);
String queueName = "testQueue";
// Run a MR job
// create a MR cluster
conf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
conf.set("mapred.queue.names", queueName);
MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, new JobConf(conf));
// run a job
Path inDir = new Path(tempDir, "input");
Path outDir = new Path(tempDir, "output");
JobHistoryParser parser = null;
RewindableInputStream ris = null;
ArrayList<String> seenEvents = new ArrayList<String>(10);
RunningJob rJob = null;
try {
JobConf jobConf = mrCluster.createJobConf();
jobConf.setQueueName(queueName);
// construct a job with 1 map and 1 reduce task.
rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 1);
rJob.waitForCompletion();
assertTrue("Job failed", rJob.isSuccessful());
JobID id = rJob.getID();
// get the jobhistory filepath
Path inputPath =
new Path(JobHistory.getHistoryFilePath(org.apache.hadoop.mapred.JobID.downgrade(id)));
// wait for 10 secs for the jobhistory file to move into the done folder
for (int i = 0; i < 100; ++i) {
if (lfs.exists(inputPath)) {
break;
}
TimeUnit.MILLISECONDS.wait(100);
}
assertTrue("Missing job history file", lfs.exists(inputPath));
InputDemuxer inputDemuxer = new DefaultInputDemuxer();
inputDemuxer.bindTo(inputPath, conf);
Pair<String, InputStream> filePair = inputDemuxer.getNext();
assertNotNull(filePair);
ris = new RewindableInputStream(filePair.second());
// Test if the JobHistoryParserFactory can detect the parser correctly
parser = JobHistoryParserFactory.getParser(ris);
// Get ParsedJob
String jobId = TraceBuilder.extractJobID(filePair.first());
JobBuilder builder = new JobBuilder(jobId);
HistoryEvent e;
while ((e = parser.nextEvent()) != null) {
String eventString = e.getEventType().toString();
System.out.println(eventString);
seenEvents.add(eventString);
if (builder != null) {
builder.process(e);
}
}
ParsedJob parsedJob = builder.build();
// validate the obtainXXX api of ParsedJob, ParsedTask and
// ParsedTaskAttempt.
validateParsedJob(parsedJob, 1, 1, queueName);
} finally {
// stop the MR cluster
mrCluster.shutdown();
if (ris != null) {
ris.close();
}
if (parser != null) {
parser.close();
}
// cleanup the filesystem
lfs.delete(tempDir, true);
}
// Check against the gold standard
System.out.println("testCurrentJHParser validating using gold std ");
String[] goldLines =
new String[] {
"JOB_SUBMITTED",
"JOB_PRIORITY_CHANGED",
"JOB_STATUS_CHANGED",
"JOB_INITED",
"JOB_INFO_CHANGED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"JOB_STATUS_CHANGED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"REDUCE_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"JOB_STATUS_CHANGED",
"JOB_FINISHED"
};
// Check the output with gold std
assertEquals("Size mismatch", goldLines.length, seenEvents.size());
int index = 0;
for (String goldLine : goldLines) {
assertEquals("Content mismatch", goldLine, seenEvents.get(index++));
}
}
@Test
public void testHadoop20JHParser() throws Exception {
// Disabled
if (true) return;
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
boolean success = false;
final Path rootInputDir =
new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
final Path tempDir = new Path(rootTempDir, "TestHadoop20JHParser");
lfs.delete(tempDir, true);
final Path inputPath = new Path(rootInputPath, "v20-single-input-log.gz");
final Path goldPath = new Path(rootInputPath, "v20-single-input-log-event-classes.text.gz");
InputStream inputLogStream = new PossiblyDecompressedInputStream(inputPath, conf);
InputStream inputGoldStream = new PossiblyDecompressedInputStream(goldPath, conf);
BufferedInputStream bis = new BufferedInputStream(inputLogStream);
bis.mark(10000);
Hadoop20JHParser parser = new Hadoop20JHParser(bis);
final Path resultPath = new Path(tempDir, "result.text");
System.out.println("testHadoop20JHParser sent its output to " + resultPath);
Compressor compressor;
FileSystem fs = resultPath.getFileSystem(conf);
CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(resultPath);
OutputStream output;
if (codec != null) {
compressor = CodecPool.getCompressor(codec);
output = codec.createOutputStream(fs.create(resultPath), compressor);
} else {
output = fs.create(resultPath);
}
PrintStream printStream = new PrintStream(output);
try {
assertEquals(
"Hadoop20JHParser can't parse the test file",
true,
Hadoop20JHParser.canParse(inputLogStream));
bis.reset();
HistoryEvent event = parser.nextEvent();
while (event != null) {
printStream.println(event.getClass().getCanonicalName());
event = parser.nextEvent();
}
printStream.close();
LineReader goldLines = new LineReader(inputGoldStream);
LineReader resultLines =
new LineReader(new PossiblyDecompressedInputStream(resultPath, conf));
int lineNumber = 1;
try {
Text goldLine = new Text();
Text resultLine = new Text();
int goldRead = goldLines.readLine(goldLine);
int resultRead = resultLines.readLine(resultLine);
while (goldRead * resultRead != 0) {
if (!goldLine.equals(resultLine)) {
assertEquals("Type mismatch detected", goldLine, resultLine);
break;
}
goldRead = goldLines.readLine(goldLine);
resultRead = resultLines.readLine(resultLine);
++lineNumber;
}
if (goldRead != resultRead) {
assertEquals(
"the "
+ (goldRead > resultRead ? "gold" : resultRead)
+ " file contains more text at line "
+ lineNumber,
goldRead,
resultRead);
}
success = true;
} finally {
goldLines.close();
resultLines.close();
if (success) {
lfs.delete(resultPath, false);
}
}
} finally {
if (parser == null) {
inputLogStream.close();
} else {
if (parser != null) {
parser.close();
}
}
if (inputGoldStream != null) {
inputGoldStream.close();
}
// it's okay to do this twice [if we get an error on input]
printStream.close();
}
}
/*
* test EventReader EventReader should read the list of events and return
* instance of HistoryEvent Different HistoryEvent should have a different
* datum.
*/
@Test(timeout = 10000)
public void testEvents() throws Exception {
EventReader reader =
new EventReader(new DataInputStream(new ByteArrayInputStream(getEvents())));
HistoryEvent e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.JOB_PRIORITY_CHANGED));
assertEquals("ID", ((JobPriorityChange) e.getDatum()).getJobid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.JOB_STATUS_CHANGED));
assertEquals("ID", ((JobStatusChanged) e.getDatum()).getJobid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.TASK_UPDATED));
assertEquals("ID", ((TaskUpdated) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.JOB_KILLED));
assertEquals("ID", ((JobUnsuccessfulCompletion) e.getDatum()).getJobid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_STARTED));
assertEquals(taskId, ((TaskAttemptStarted) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_FINISHED));
assertEquals(taskId, ((TaskAttemptFinished) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_STARTED));
assertEquals(taskId, ((TaskAttemptStarted) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_FINISHED));
assertEquals(taskId, ((TaskAttemptFinished) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString());
e = reader.getNextEvent();
assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString());
reader.close();
}