public Compressor getCompressor() { CompressionCodec codec = getCodec(conf); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (LOG.isTraceEnabled()) LOG.trace("Retrieved compressor " + compressor + " from pool."); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using it. LOG.warn("Compressor obtained from CodecPool is already finished()"); } compressor.reset(); } return compressor; } return null; }
public Compressor getCompressor() { CompressionCodec codec = getCodec(conf); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool is already finished()"); // throw new AssertionError( // "Compressor obtained from CodecPool is already finished()"); } compressor.reset(); } return compressor; } return null; }
@Test public void testHadoop20JHParser() throws Exception { // Disabled if (true) return; final Configuration conf = new Configuration(); final FileSystem lfs = FileSystem.getLocal(conf); boolean success = false; final Path rootInputDir = new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs); final Path rootTempDir = new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs); final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test"); final Path tempDir = new Path(rootTempDir, "TestHadoop20JHParser"); lfs.delete(tempDir, true); final Path inputPath = new Path(rootInputPath, "v20-single-input-log.gz"); final Path goldPath = new Path(rootInputPath, "v20-single-input-log-event-classes.text.gz"); InputStream inputLogStream = new PossiblyDecompressedInputStream(inputPath, conf); InputStream inputGoldStream = new PossiblyDecompressedInputStream(goldPath, conf); BufferedInputStream bis = new BufferedInputStream(inputLogStream); bis.mark(10000); Hadoop20JHParser parser = new Hadoop20JHParser(bis); final Path resultPath = new Path(tempDir, "result.text"); System.out.println("testHadoop20JHParser sent its output to " + resultPath); Compressor compressor; FileSystem fs = resultPath.getFileSystem(conf); CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(resultPath); OutputStream output; if (codec != null) { compressor = CodecPool.getCompressor(codec); output = codec.createOutputStream(fs.create(resultPath), compressor); } else { output = fs.create(resultPath); } PrintStream printStream = new PrintStream(output); try { assertEquals( "Hadoop20JHParser can't parse the test file", true, Hadoop20JHParser.canParse(inputLogStream)); bis.reset(); HistoryEvent event = parser.nextEvent(); while (event != null) { printStream.println(event.getClass().getCanonicalName()); event = parser.nextEvent(); } printStream.close(); LineReader goldLines = new LineReader(inputGoldStream); LineReader resultLines = new LineReader(new PossiblyDecompressedInputStream(resultPath, conf)); int lineNumber = 1; try { Text goldLine = new Text(); Text resultLine = new Text(); int goldRead = goldLines.readLine(goldLine); int resultRead = resultLines.readLine(resultLine); while (goldRead * resultRead != 0) { if (!goldLine.equals(resultLine)) { assertEquals("Type mismatch detected", goldLine, resultLine); break; } goldRead = goldLines.readLine(goldLine); resultRead = resultLines.readLine(resultLine); ++lineNumber; } if (goldRead != resultRead) { assertEquals( "the " + (goldRead > resultRead ? "gold" : resultRead) + " file contains more text at line " + lineNumber, goldRead, resultRead); } success = true; } finally { goldLines.close(); resultLines.close(); if (success) { lfs.delete(resultPath, false); } } } finally { if (parser == null) { inputLogStream.close(); } else { if (parser != null) { parser.close(); } } if (inputGoldStream != null) { inputGoldStream.close(); } // it's okay to do this twice [if we get an error on input] printStream.close(); } }