/** {@inheritDoc} */ @Override protected void configureJob(Job job) throws IOException { final Configuration conf = job.getConfiguration(); // Construct the producer instance. if (null == mProducerClass) { throw new JobConfigurationException("Must specify a producer."); } mProducer = ReflectionUtils.newInstance(mProducerClass, job.getConfiguration()); mDataRequest = mProducer.getDataRequest(); // Serialize the producer class name into the job configuration. conf.setClass(KijiProducer.CONF_PRODUCER_CLASS, mProducerClass, KijiProducer.class); // Configure the mapper and reducer to use. Preconditions.checkState(getJobOutput() instanceof KijiTableMapReduceJobOutput); // Write to the table, but make sure the output table is the same as the input table. if (!getInputTable().equals(((KijiTableMapReduceJobOutput) getJobOutput()).getTable())) { throw new JobConfigurationException("Output table must be the same as the input table."); } // Producers should output to HFiles. mMapper = new ProduceMapper(); mReducer = new IdentityReducer<Object, Object>(); // Set the compression type for the generated HFiles to match the target locality group. final String outputColumn = mProducer.getOutputColumn(); final LocalityGroupLayout targetLocalityGroup = getTableLayout() .getFamilyMap() .get(new KijiColumnName(outputColumn).getFamily()) .getLocalityGroup(); conf.set( KijiHFileOutputFormat.CONF_HFILE_COMPRESSION, targetLocalityGroup.getDesc().getCompressionType().name().toLowerCase(Locale.US)); job.setJobName("Kiji produce: " + mProducerClass.getSimpleName()); // Configure the table input job. super.configureJob(job); }
/** * Constructor. * * @param context Task attempt context. * @param lgLayout Layout of the locality group. * @throws IOException on I/O error. */ public LocalityGroupRecordWriter(TaskAttemptContext context, LocalityGroupLayout lgLayout) throws IOException { mLGLayout = Preconditions.checkNotNull(lgLayout); mFamily = lgLayout.getId().toString(); // These parameters might be specific to each locality group: mMaxFileSizeBytes = mConf.getLong(CONF_HREGION_MAX_FILESIZE, DEFAULT_HREGION_MAX_FILESIZE); mBlockSizeBytes = mConf.getInt(CONF_HFILE_BLOCKSIZE, DEFAULT_HFILE_BLOCKSIZE); mFamilyDir = new Path(mOutputDir, mFamily); if (!mFileSystem.exists(mFamilyDir)) { if (!mFileSystem.mkdirs(mFamilyDir)) { throw new IOException( String.format("Unable to create output directory: %s", mFamilyDir)); } } mCompressionType = Compression.getCompressionAlgorithmByName( mLGLayout.getDesc().getCompressionType().toString().toLowerCase(Locale.ROOT)); mWriter = openNewWriter(); }