/** Test if {@link CurrentJHParser} can read events from current JH files. */ @Test public void testCurrentJHParser() throws Exception { final Configuration conf = new Configuration(); final FileSystem lfs = FileSystem.getLocal(conf); final Path rootTempDir = new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs); final Path tempDir = new Path(rootTempDir, "TestCurrentJHParser"); lfs.delete(tempDir, true); String queueName = "testQueue"; // Run a MR job // create a MR cluster conf.setInt("mapred.tasktracker.map.tasks.maximum", 1); conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1); conf.set("mapred.queue.names", queueName); MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, new JobConf(conf)); // run a job Path inDir = new Path(tempDir, "input"); Path outDir = new Path(tempDir, "output"); JobHistoryParser parser = null; RewindableInputStream ris = null; ArrayList<String> seenEvents = new ArrayList<String>(10); RunningJob rJob = null; try { JobConf jobConf = mrCluster.createJobConf(); jobConf.setQueueName(queueName); // construct a job with 1 map and 1 reduce task. rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 1); rJob.waitForCompletion(); assertTrue("Job failed", rJob.isSuccessful()); JobID id = rJob.getID(); // get the jobhistory filepath Path inputPath = new Path(JobHistory.getHistoryFilePath(org.apache.hadoop.mapred.JobID.downgrade(id))); // wait for 10 secs for the jobhistory file to move into the done folder for (int i = 0; i < 100; ++i) { if (lfs.exists(inputPath)) { break; } TimeUnit.MILLISECONDS.wait(100); } assertTrue("Missing job history file", lfs.exists(inputPath)); InputDemuxer inputDemuxer = new DefaultInputDemuxer(); inputDemuxer.bindTo(inputPath, conf); Pair<String, InputStream> filePair = inputDemuxer.getNext(); assertNotNull(filePair); ris = new RewindableInputStream(filePair.second()); // Test if the JobHistoryParserFactory can detect the parser correctly parser = JobHistoryParserFactory.getParser(ris); // Get ParsedJob String jobId = TraceBuilder.extractJobID(filePair.first()); JobBuilder builder = new JobBuilder(jobId); HistoryEvent e; while ((e = parser.nextEvent()) != null) { String eventString = e.getEventType().toString(); System.out.println(eventString); seenEvents.add(eventString); if (builder != null) { builder.process(e); } } ParsedJob parsedJob = builder.build(); // validate the obtainXXX api of ParsedJob, ParsedTask and // ParsedTaskAttempt. validateParsedJob(parsedJob, 1, 1, queueName); } finally { // stop the MR cluster mrCluster.shutdown(); if (ris != null) { ris.close(); } if (parser != null) { parser.close(); } // cleanup the filesystem lfs.delete(tempDir, true); } // Check against the gold standard System.out.println("testCurrentJHParser validating using gold std "); String[] goldLines = new String[] { "JOB_SUBMITTED", "JOB_PRIORITY_CHANGED", "JOB_STATUS_CHANGED", "JOB_INITED", "JOB_INFO_CHANGED", "TASK_STARTED", "MAP_ATTEMPT_STARTED", "MAP_ATTEMPT_FINISHED", "MAP_ATTEMPT_FINISHED", "TASK_UPDATED", "TASK_FINISHED", "JOB_STATUS_CHANGED", "TASK_STARTED", "MAP_ATTEMPT_STARTED", "MAP_ATTEMPT_FINISHED", "MAP_ATTEMPT_FINISHED", "TASK_UPDATED", "TASK_FINISHED", "TASK_STARTED", "MAP_ATTEMPT_STARTED", "MAP_ATTEMPT_FINISHED", "REDUCE_ATTEMPT_FINISHED", "TASK_UPDATED", "TASK_FINISHED", "TASK_STARTED", "MAP_ATTEMPT_STARTED", "MAP_ATTEMPT_FINISHED", "MAP_ATTEMPT_FINISHED", "TASK_UPDATED", "TASK_FINISHED", "JOB_STATUS_CHANGED", "JOB_FINISHED" }; // Check the output with gold std assertEquals("Size mismatch", goldLines.length, seenEvents.size()); int index = 0; for (String goldLine : goldLines) { assertEquals("Content mismatch", goldLine, seenEvents.get(index++)); } }