@Override
protected void configureJob(Job job) throws IOException {
Configuration conf = job.getConfiguration();
job.setJarByClass(PartialBuilder.class);
FileInputFormat.setInputPaths(job, getDataPath());
FileOutputFormat.setOutputPath(job, getOutputPath(conf));
job.setOutputKeyClass(TreeID.class);
job.setOutputValueClass(MapredOutput.class);
job.setMapperClass(Step1Mapper.class);
job.setNumReduceTasks(0); // no reducers
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
// For this implementation to work, mapred.map.tasks needs to be set to the actual
// number of mappers Hadoop will use:
TextInputFormat inputFormat = new TextInputFormat();
List<?> splits = inputFormat.getSplits(job);
if (splits == null || splits.isEmpty()) {
log.warn("Unable to compute number of splits?");
} else {
int numSplits = splits.size();
log.info("Setting mapred.map.tasks = {}", numSplits);
conf.setInt("mapred.map.tasks", numSplits);
}
}
@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
return textIF.getSplits(context);
}
