/** {@inheritDoc} */
@Override
protected void configureJob(Job job) throws IOException {
final Configuration conf = job.getConfiguration();
// Construct the producer instance.
if (null == mProducerClass) {
throw new JobConfigurationException("Must specify a producer.");
}
mProducer = ReflectionUtils.newInstance(mProducerClass, job.getConfiguration());
mDataRequest = mProducer.getDataRequest();
// Serialize the producer class name into the job configuration.
conf.setClass(KijiProducer.CONF_PRODUCER_CLASS, mProducerClass, KijiProducer.class);
// Configure the mapper and reducer to use.
Preconditions.checkState(getJobOutput() instanceof KijiTableMapReduceJobOutput);
// Write to the table, but make sure the output table is the same as the input table.
if (!getInputTable().equals(((KijiTableMapReduceJobOutput) getJobOutput()).getTable())) {
throw new JobConfigurationException("Output table must be the same as the input table.");
}
// Producers should output to HFiles.
mMapper = new ProduceMapper();
mReducer = new IdentityReducer<Object, Object>();
// Set the compression type for the generated HFiles to match the target locality group.
final String outputColumn = mProducer.getOutputColumn();
final LocalityGroupLayout targetLocalityGroup =
getTableLayout()
.getFamilyMap()
.get(new KijiColumnName(outputColumn).getFamily())
.getLocalityGroup();
conf.set(
KijiHFileOutputFormat.CONF_HFILE_COMPRESSION,
targetLocalityGroup.getDesc().getCompressionType().name().toLowerCase(Locale.US));
job.setJobName("Kiji produce: " + mProducerClass.getSimpleName());
// Configure the table input job.
super.configureJob(job);
}
/**
* Constructor.
*
* @param context Task attempt context.
* @param lgLayout Layout of the locality group.
* @throws IOException on I/O error.
*/
public LocalityGroupRecordWriter(TaskAttemptContext context, LocalityGroupLayout lgLayout)
throws IOException {
mLGLayout = Preconditions.checkNotNull(lgLayout);
mFamily = lgLayout.getId().toString();
// These parameters might be specific to each locality group:
mMaxFileSizeBytes = mConf.getLong(CONF_HREGION_MAX_FILESIZE, DEFAULT_HREGION_MAX_FILESIZE);
mBlockSizeBytes = mConf.getInt(CONF_HFILE_BLOCKSIZE, DEFAULT_HFILE_BLOCKSIZE);
mFamilyDir = new Path(mOutputDir, mFamily);
if (!mFileSystem.exists(mFamilyDir)) {
if (!mFileSystem.mkdirs(mFamilyDir)) {
throw new IOException(
String.format("Unable to create output directory: %s", mFamilyDir));
}
}
mCompressionType =
Compression.getCompressionAlgorithmByName(
mLGLayout.getDesc().getCompressionType().toString().toLowerCase(Locale.ROOT));
mWriter = openNewWriter();
}
