public JobBuilder compressor(CompressionType type, Class<? extends CompressionCodec> codec)
throws IOException {
_jobConf.setBoolean("mapred.output.compress", true);
_jobConf.set("mapred.output.compression.type", type.toString());
_jobConf.setClass("mapred.output.compression.codec", codec, CompressionCodec.class);
return this;
}
public JobBuilder compressType(CompressionType type) throws IOException {
if (type == CompressionType.NONE) {
_jobConf.setBoolean("mapred.output.compress", false);
} else {
_jobConf.setBoolean("mapred.output.compress", true);
}
_jobConf.set("mapred.output.compression.type", type.toString());
return this;
}
/**
* 圧縮の種類を取得する。
*
* @param strCompType CompressionTypeの文字列
* @return CompressionType
*/
protected CompressionType getCompType(String strCompType) {
CompressionType compType = null;
try {
compType = CompressionType.valueOf(strCompType);
} catch (Exception e) {
compType = CompressionType.NONE;
LOG.warn("TG-EXTRACTOR-02004", strCompType);
}
return compType;
}
public static RecordWriter getHiveRecordWriter(
JobConf jc,
TableDesc tableInfo,
Class<? extends Writable> outputClass,
FileSinkDesc conf,
Path outPath)
throws HiveException {
try {
HiveOutputFormat<?, ?> hiveOutputFormat = tableInfo.getOutputFileFormatClass().newInstance();
boolean isCompressed = conf.getCompressed();
JobConf jc_output = jc;
if (isCompressed) {
jc_output = new JobConf(jc);
String codecStr = conf.getCompressCodec();
if (codecStr != null && !codecStr.trim().equals("")) {
Class<? extends CompressionCodec> codec =
(Class<? extends CompressionCodec>) Class.forName(codecStr);
FileOutputFormat.setOutputCompressorClass(jc_output, codec);
}
String type = conf.getCompressType();
if (type != null && !type.trim().equals("")) {
CompressionType style = CompressionType.valueOf(type);
SequenceFileOutputFormat.setOutputCompressionType(jc, style);
}
}
return getRecordWriter(
jc_output,
hiveOutputFormat,
outputClass,
isCompressed,
tableInfo.getProperties(),
outPath);
} catch (Exception e) {
throw new HiveException(e);
}
}
