OrcRecordWriter(Path path, OrcFile.WriterOptions options, JobConf conf) {
this.path = path;
this.options = options;
/*to enable override compression properties from job conf*/
String compression = conf.get(OrcFile.COMPRESSION);
if (compression != null) {
this.options.compress(CompressionKind.valueOf(compression));
}
String compressSize = conf.get(OrcFile.COMPRESSION_BLOCK_SIZE);
if (compressSize != null && !"".equals(compressSize.trim())) {
this.options.bufferSize(Integer.parseInt(compressSize));
}
String stripeSize = conf.get(OrcFile.STRIPE_SIZE);
if (stripeSize != null && !"".equals(stripeSize.trim())) {
this.options.stripeSize(Long.parseLong(stripeSize));
}
String strideSize = conf.get(OrcFile.ROW_INDEX_STRIDE);
if (strideSize != null && !"".equals(strideSize.trim())) {
this.options.rowIndexStride(Integer.parseInt(strideSize));
}
String enableIndexes = conf.get(OrcFile.ENABLE_INDEXES);
if (enableIndexes != null && "false".equals(enableIndexes)) {
this.options.rowIndexStride(0);
}
String blockPadding = conf.get(OrcFile.BLOCK_PADDING);
if (blockPadding != null && !"".equals(blockPadding)) {
this.options.blockPadding(Boolean.valueOf(blockPadding));
}
}
MetaInfoObjExtractor(String codecStr, int bufferSize, int metadataSize, ByteBuffer footerBuffer)
throws IOException {
this.compressionKind = CompressionKind.valueOf(codecStr);
this.bufferSize = bufferSize;
this.codec = WriterImpl.createCodec(compressionKind);
this.metadataSize = metadataSize;
int position = footerBuffer.position();
int footerBufferSize = footerBuffer.limit() - footerBuffer.position() - metadataSize;
footerBuffer.limit(position + metadataSize);
InputStream instream =
InStream.create(
"metadata",
Lists.<DiskRange>newArrayList(new BufferChunk(footerBuffer, 0)),
metadataSize,
codec,
bufferSize);
this.metadata = OrcProto.Metadata.parseFrom(instream);
footerBuffer.position(position + metadataSize);
footerBuffer.limit(position + metadataSize + footerBufferSize);
instream =
InStream.create(
"footer",
Lists.<DiskRange>newArrayList(new BufferChunk(footerBuffer, 0)),
footerBufferSize,
codec,
bufferSize);
this.footer = OrcProto.Footer.parseFrom(instream);
footerBuffer.position(position);
this.inspector = OrcStruct.createObjectInspector(0, footer.getTypesList());
}
public FileMetaInfo getFileMetaInfo() {
return new FileMetaInfo(
compressionKind.toString(),
bufferSize,
metadataSize,
footerByteBuffer,
versionList,
writerVersion);
}
@Override
public FileSinkOperator.RecordWriter getHiveRecordWriter(
JobConf conf,
Path path,
Class<? extends Writable> valueClass,
boolean isCompressed,
Properties tableProperties,
Progressable reporter)
throws IOException {
OrcFile.WriterOptions options = OrcFile.writerOptions(conf);
if (tableProperties.containsKey(OrcFile.STRIPE_SIZE)) {
options.stripeSize(Long.parseLong(tableProperties.getProperty(OrcFile.STRIPE_SIZE)));
}
if (tableProperties.containsKey(OrcFile.COMPRESSION)) {
options.compress(CompressionKind.valueOf(tableProperties.getProperty(OrcFile.COMPRESSION)));
}
if (tableProperties.containsKey(OrcFile.COMPRESSION_BLOCK_SIZE)) {
options.bufferSize(
Integer.parseInt(tableProperties.getProperty(OrcFile.COMPRESSION_BLOCK_SIZE)));
}
if (tableProperties.containsKey(OrcFile.ROW_INDEX_STRIDE)) {
options.rowIndexStride(
Integer.parseInt(tableProperties.getProperty(OrcFile.ROW_INDEX_STRIDE)));
}
if (tableProperties.containsKey(OrcFile.ENABLE_INDEXES)) {
if ("false".equals(tableProperties.getProperty(OrcFile.ENABLE_INDEXES))) {
options.rowIndexStride(0);
}
}
if (tableProperties.containsKey(OrcFile.BLOCK_PADDING)) {
options.blockPadding(
Boolean.parseBoolean(tableProperties.getProperty(OrcFile.BLOCK_PADDING)));
}
return new OrcRecordWriter(path, options, conf);
}
