public void checkOutputSpecs(FileSystem fs, JobConf conf) throws IOException {
Args args = (Args) Utils.getObject(conf, ARGS_CONF);
fs = Utils.getFS(args.outputDirHdfs, conf);
if (conf.getBoolean("mapred.reduce.tasks.speculative.execution", true)) {
// Because we don't want to write a bunch of extra times.
throw new InvalidJobConfException("Speculative execution should be false");
}
if (fs.exists(new Path(args.outputDirHdfs))) {
throw new InvalidJobConfException("Output dir already exists " + args.outputDirHdfs);
}
}
public ElephantRecordWriter(Configuration conf, Args args, Progressable progressable)
throws IOException {
fileSystem = Utils.getFS(args.outputDirHdfs, conf);
this.args = args;
this.progressable = progressable;
localManager =
new LocalElephantManager(fileSystem, args.spec, LocalElephantManager.getTmpDirs(conf));
}
public RecordWriter<IntWritable, ElephantRecordWritable> getRecordWriter(
FileSystem fs, JobConf conf, String string, Progressable progressable) throws IOException {
return new ElephantRecordWriter(conf, (Args) Utils.getObject(conf, ARGS_CONF), progressable);
}