MetaInfoObjExtractor(String codecStr, int bufferSize, int metadataSize, ByteBuffer footerBuffer)
throws IOException {
this.compressionKind = CompressionKind.valueOf(codecStr);
this.bufferSize = bufferSize;
this.codec = WriterImpl.createCodec(compressionKind);
this.metadataSize = metadataSize;
int position = footerBuffer.position();
int footerBufferSize = footerBuffer.limit() - footerBuffer.position() - metadataSize;
footerBuffer.limit(position + metadataSize);
InputStream instream =
InStream.create(
"metadata",
Lists.<DiskRange>newArrayList(new BufferChunk(footerBuffer, 0)),
metadataSize,
codec,
bufferSize);
this.metadata = OrcProto.Metadata.parseFrom(instream);
footerBuffer.position(position + metadataSize);
footerBuffer.limit(position + metadataSize + footerBufferSize);
instream =
InStream.create(
"footer",
Lists.<DiskRange>newArrayList(new BufferChunk(footerBuffer, 0)),
footerBufferSize,
codec,
bufferSize);
this.footer = OrcProto.Footer.parseFrom(instream);
footerBuffer.position(position);
this.inspector = OrcStruct.createObjectInspector(0, footer.getTypesList());
}
public ReaderImpl(FileSystem fs, Path path, Configuration conf) throws IOException {
this.fileSystem = fs;
this.path = path;
this.conf = conf;
FSDataInputStream file = fs.open(path);
long size = fs.getFileStatus(path).getLen();
int readSize = (int) Math.min(size, DIRECTORY_SIZE_GUESS);
ByteBuffer buffer = ByteBuffer.allocate(readSize);
InStream.read(
file,
size - readSize,
buffer.array(),
buffer.arrayOffset() + buffer.position(),
buffer.remaining());
int psLen = buffer.get(readSize - 1);
int psOffset = readSize - 1 - psLen;
CodedInputStream in =
CodedInputStream.newInstance(buffer.array(), buffer.arrayOffset() + psOffset, psLen);
OrcProto.PostScript ps = OrcProto.PostScript.parseFrom(in);
int footerSize = (int) ps.getFooterLength();
bufferSize = (int) ps.getCompressionBlockSize();
switch (ps.getCompression()) {
case NONE:
compressionKind = CompressionKind.NONE;
break;
case ZLIB:
compressionKind = CompressionKind.ZLIB;
break;
case SNAPPY:
compressionKind = CompressionKind.SNAPPY;
break;
case LZO:
compressionKind = CompressionKind.LZO;
break;
default:
throw new IllegalArgumentException("Unknown compression");
}
codec = WriterImpl.createCodec(compressionKind);
InputStream instream =
InStream.create(
"footer", file, size - 1 - psLen - footerSize, footerSize, codec, bufferSize);
footer = OrcProto.Footer.parseFrom(instream);
inspector = new OrcLazyRowObjectInspector(0, footer.getTypesList());
file.close();
}
