@Test public void testHadoop20JHParser() throws Exception { // Disabled if (true) return; final Configuration conf = new Configuration(); final FileSystem lfs = FileSystem.getLocal(conf); boolean success = false; final Path rootInputDir = new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs); final Path rootTempDir = new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs); final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test"); final Path tempDir = new Path(rootTempDir, "TestHadoop20JHParser"); lfs.delete(tempDir, true); final Path inputPath = new Path(rootInputPath, "v20-single-input-log.gz"); final Path goldPath = new Path(rootInputPath, "v20-single-input-log-event-classes.text.gz"); InputStream inputLogStream = new PossiblyDecompressedInputStream(inputPath, conf); InputStream inputGoldStream = new PossiblyDecompressedInputStream(goldPath, conf); BufferedInputStream bis = new BufferedInputStream(inputLogStream); bis.mark(10000); Hadoop20JHParser parser = new Hadoop20JHParser(bis); final Path resultPath = new Path(tempDir, "result.text"); System.out.println("testHadoop20JHParser sent its output to " + resultPath); Compressor compressor; FileSystem fs = resultPath.getFileSystem(conf); CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(resultPath); OutputStream output; if (codec != null) { compressor = CodecPool.getCompressor(codec); output = codec.createOutputStream(fs.create(resultPath), compressor); } else { output = fs.create(resultPath); } PrintStream printStream = new PrintStream(output); try { assertEquals( "Hadoop20JHParser can't parse the test file", true, Hadoop20JHParser.canParse(inputLogStream)); bis.reset(); HistoryEvent event = parser.nextEvent(); while (event != null) { printStream.println(event.getClass().getCanonicalName()); event = parser.nextEvent(); } printStream.close(); LineReader goldLines = new LineReader(inputGoldStream); LineReader resultLines = new LineReader(new PossiblyDecompressedInputStream(resultPath, conf)); int lineNumber = 1; try { Text goldLine = new Text(); Text resultLine = new Text(); int goldRead = goldLines.readLine(goldLine); int resultRead = resultLines.readLine(resultLine); while (goldRead * resultRead != 0) { if (!goldLine.equals(resultLine)) { assertEquals("Type mismatch detected", goldLine, resultLine); break; } goldRead = goldLines.readLine(goldLine); resultRead = resultLines.readLine(resultLine); ++lineNumber; } if (goldRead != resultRead) { assertEquals( "the " + (goldRead > resultRead ? "gold" : resultRead) + " file contains more text at line " + lineNumber, goldRead, resultRead); } success = true; } finally { goldLines.close(); resultLines.close(); if (success) { lfs.delete(resultPath, false); } } } finally { if (parser == null) { inputLogStream.close(); } else { if (parser != null) { parser.close(); } } if (inputGoldStream != null) { inputGoldStream.close(); } // it's okay to do this twice [if we get an error on input] printStream.close(); } }
/** Test if {@link CurrentJHParser} can read events from current JH files. */ @Test public void testCurrentJHParser() throws Exception { final Configuration conf = new Configuration(); final FileSystem lfs = FileSystem.getLocal(conf); final Path rootTempDir = new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs); final Path tempDir = new Path(rootTempDir, "TestCurrentJHParser"); lfs.delete(tempDir, true); String queueName = "testQueue"; // Run a MR job // create a MR cluster conf.setInt("mapred.tasktracker.map.tasks.maximum", 1); conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1); conf.set("mapred.queue.names", queueName); MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, new JobConf(conf)); // run a job Path inDir = new Path(tempDir, "input"); Path outDir = new Path(tempDir, "output"); JobHistoryParser parser = null; RewindableInputStream ris = null; ArrayList<String> seenEvents = new ArrayList<String>(10); RunningJob rJob = null; try { JobConf jobConf = mrCluster.createJobConf(); jobConf.setQueueName(queueName); // construct a job with 1 map and 1 reduce task. rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 1); rJob.waitForCompletion(); assertTrue("Job failed", rJob.isSuccessful()); JobID id = rJob.getID(); // get the jobhistory filepath Path inputPath = new Path(JobHistory.getHistoryFilePath(org.apache.hadoop.mapred.JobID.downgrade(id))); // wait for 10 secs for the jobhistory file to move into the done folder for (int i = 0; i < 100; ++i) { if (lfs.exists(inputPath)) { break; } TimeUnit.MILLISECONDS.wait(100); } assertTrue("Missing job history file", lfs.exists(inputPath)); InputDemuxer inputDemuxer = new DefaultInputDemuxer(); inputDemuxer.bindTo(inputPath, conf); Pair<String, InputStream> filePair = inputDemuxer.getNext(); assertNotNull(filePair); ris = new RewindableInputStream(filePair.second()); // Test if the JobHistoryParserFactory can detect the parser correctly parser = JobHistoryParserFactory.getParser(ris); // Get ParsedJob String jobId = TraceBuilder.extractJobID(filePair.first()); JobBuilder builder = new JobBuilder(jobId); HistoryEvent e; while ((e = parser.nextEvent()) != null) { String eventString = e.getEventType().toString(); System.out.println(eventString); seenEvents.add(eventString); if (builder != null) { builder.process(e); } } ParsedJob parsedJob = builder.build(); // validate the obtainXXX api of ParsedJob, ParsedTask and // ParsedTaskAttempt. validateParsedJob(parsedJob, 1, 1, queueName); } finally { // stop the MR cluster mrCluster.shutdown(); if (ris != null) { ris.close(); } if (parser != null) { parser.close(); } // cleanup the filesystem lfs.delete(tempDir, true); } // Check against the gold standard System.out.println("testCurrentJHParser validating using gold std "); String[] goldLines = new String[] { "JOB_SUBMITTED", "JOB_PRIORITY_CHANGED", "JOB_STATUS_CHANGED", "JOB_INITED", "JOB_INFO_CHANGED", "TASK_STARTED", "MAP_ATTEMPT_STARTED", "MAP_ATTEMPT_FINISHED", "MAP_ATTEMPT_FINISHED", "TASK_UPDATED", "TASK_FINISHED", "JOB_STATUS_CHANGED", "TASK_STARTED", "MAP_ATTEMPT_STARTED", "MAP_ATTEMPT_FINISHED", "MAP_ATTEMPT_FINISHED", "TASK_UPDATED", "TASK_FINISHED", "TASK_STARTED", "MAP_ATTEMPT_STARTED", "MAP_ATTEMPT_FINISHED", "REDUCE_ATTEMPT_FINISHED", "TASK_UPDATED", "TASK_FINISHED", "TASK_STARTED", "MAP_ATTEMPT_STARTED", "MAP_ATTEMPT_FINISHED", "MAP_ATTEMPT_FINISHED", "TASK_UPDATED", "TASK_FINISHED", "JOB_STATUS_CHANGED", "JOB_FINISHED" }; // Check the output with gold std assertEquals("Size mismatch", goldLines.length, seenEvents.size()); int index = 0; for (String goldLine : goldLines) { assertEquals("Content mismatch", goldLine, seenEvents.get(index++)); } }
/* * test EventReader EventReader should read the list of events and return * instance of HistoryEvent Different HistoryEvent should have a different * datum. */ @Test(timeout = 10000) public void testEvents() throws Exception { EventReader reader = new EventReader(new DataInputStream(new ByteArrayInputStream(getEvents()))); HistoryEvent e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.JOB_PRIORITY_CHANGED)); assertEquals("ID", ((JobPriorityChange) e.getDatum()).getJobid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.JOB_STATUS_CHANGED)); assertEquals("ID", ((JobStatusChanged) e.getDatum()).getJobid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.TASK_UPDATED)); assertEquals("ID", ((TaskUpdated) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED)); assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.JOB_KILLED)); assertEquals("ID", ((JobUnsuccessfulCompletion) e.getDatum()).getJobid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_STARTED)); assertEquals(taskId, ((TaskAttemptStarted) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_FINISHED)); assertEquals(taskId, ((TaskAttemptFinished) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED)); assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED)); assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_STARTED)); assertEquals(taskId, ((TaskAttemptStarted) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_FINISHED)); assertEquals(taskId, ((TaskAttemptFinished) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED)); assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString()); e = reader.getNextEvent(); assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED)); assertEquals(taskId, ((TaskAttemptUnsuccessfulCompletion) e.getDatum()).getTaskid().toString()); reader.close(); }