public Compressor getCompressor() {
CompressionCodec codec = getCodec(conf);
if (codec != null) {
Compressor compressor = CodecPool.getCompressor(codec);
if (LOG.isTraceEnabled()) LOG.trace("Retrieved compressor " + compressor + " from pool.");
if (compressor != null) {
if (compressor.finished()) {
// Somebody returns the compressor to CodecPool but is still using it.
LOG.warn("Compressor obtained from CodecPool is already finished()");
}
compressor.reset();
}
return compressor;
}
return null;
}
public Compressor getCompressor() {
CompressionCodec codec = getCodec(conf);
if (codec != null) {
Compressor compressor = CodecPool.getCompressor(codec);
if (compressor != null) {
if (compressor.finished()) {
// Somebody returns the compressor to CodecPool but is still using
// it.
LOG.warn("Compressor obtained from CodecPool is already finished()");
// throw new AssertionError(
// "Compressor obtained from CodecPool is already finished()");
}
compressor.reset();
}
return compressor;
}
return null;
}
@Test
public void testHadoop20JHParser() throws Exception {
// Disabled
if (true) return;
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
boolean success = false;
final Path rootInputDir =
new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
final Path tempDir = new Path(rootTempDir, "TestHadoop20JHParser");
lfs.delete(tempDir, true);
final Path inputPath = new Path(rootInputPath, "v20-single-input-log.gz");
final Path goldPath = new Path(rootInputPath, "v20-single-input-log-event-classes.text.gz");
InputStream inputLogStream = new PossiblyDecompressedInputStream(inputPath, conf);
InputStream inputGoldStream = new PossiblyDecompressedInputStream(goldPath, conf);
BufferedInputStream bis = new BufferedInputStream(inputLogStream);
bis.mark(10000);
Hadoop20JHParser parser = new Hadoop20JHParser(bis);
final Path resultPath = new Path(tempDir, "result.text");
System.out.println("testHadoop20JHParser sent its output to " + resultPath);
Compressor compressor;
FileSystem fs = resultPath.getFileSystem(conf);
CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(resultPath);
OutputStream output;
if (codec != null) {
compressor = CodecPool.getCompressor(codec);
output = codec.createOutputStream(fs.create(resultPath), compressor);
} else {
output = fs.create(resultPath);
}
PrintStream printStream = new PrintStream(output);
try {
assertEquals(
"Hadoop20JHParser can't parse the test file",
true,
Hadoop20JHParser.canParse(inputLogStream));
bis.reset();
HistoryEvent event = parser.nextEvent();
while (event != null) {
printStream.println(event.getClass().getCanonicalName());
event = parser.nextEvent();
}
printStream.close();
LineReader goldLines = new LineReader(inputGoldStream);
LineReader resultLines =
new LineReader(new PossiblyDecompressedInputStream(resultPath, conf));
int lineNumber = 1;
try {
Text goldLine = new Text();
Text resultLine = new Text();
int goldRead = goldLines.readLine(goldLine);
int resultRead = resultLines.readLine(resultLine);
while (goldRead * resultRead != 0) {
if (!goldLine.equals(resultLine)) {
assertEquals("Type mismatch detected", goldLine, resultLine);
break;
}
goldRead = goldLines.readLine(goldLine);
resultRead = resultLines.readLine(resultLine);
++lineNumber;
}
if (goldRead != resultRead) {
assertEquals(
"the "
+ (goldRead > resultRead ? "gold" : resultRead)
+ " file contains more text at line "
+ lineNumber,
goldRead,
resultRead);
}
success = true;
} finally {
goldLines.close();
resultLines.close();
if (success) {
lfs.delete(resultPath, false);
}
}
} finally {
if (parser == null) {
inputLogStream.close();
} else {
if (parser != null) {
parser.close();
}
}
if (inputGoldStream != null) {
inputGoldStream.close();
}
// it's okay to do this twice [if we get an error on input]
printStream.close();
}
}
