public static void main(String[] args) throws Exception {
final String NAME_NODE = "hdfs://sandbox.hortonworks.com:8020";
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(WordCount.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(NullWritable.class);
if (args.length > 2) {
job.setNumReduceTasks(Integer.parseInt(args[2]));
}
job.setMapperClass(CountMapper.class);
job.setReducerClass(CountReducer.class);
job.setJarByClass(WordCount.class);
job.setNumReduceTasks(1);
FileInputFormat.addInputPath(job, new Path(args[0] + "data/plot_summaries.txt"));
FileSystem fs = FileSystem.get(conf);
// handle (e.g. delete) existing output path
Path outputDestination = new Path(args[0] + args[1]);
if (fs.exists(outputDestination)) {
fs.delete(outputDestination, true);
}
// set output path & start job1
FileOutputFormat.setOutputPath(job, outputDestination);
int jobCompletionStatus = job.waitForCompletion(true) ? 0 : 1;
}
private void generateSaltIfNeeded(Path saltFilePath, Path secretsPath) throws Exception {
FileSystem fileSystem = FileSystem.get(saltFilePath.toUri(), getConf());
if (!fileSystem.exists(saltFilePath)) {
FSDataOutputStream outputStream = fileSystem.create(saltFilePath);
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream));
int numSaltsToGenerate = getNumSecrets(secretsPath);
System.out.printf("Generating %d salts\n", numSaltsToGenerate);
for (int i = 0; i < numSaltsToGenerate; i++) {
writer.write(BCrypt.gensalt());
writer.newLine();
}
writer.close();
outputStream.close();
}
}
