public static void main(String[] args) throws Exception {
final String NAME_NODE = "hdfs://sandbox.hortonworks.com:8020";
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(WordCount.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(NullWritable.class);
if (args.length > 2) {
job.setNumReduceTasks(Integer.parseInt(args[2]));
}
job.setMapperClass(CountMapper.class);
job.setReducerClass(CountReducer.class);
job.setJarByClass(WordCount.class);
job.setNumReduceTasks(1);
FileInputFormat.addInputPath(job, new Path(args[0] + "data/plot_summaries.txt"));
FileSystem fs = FileSystem.get(conf);
// handle (e.g. delete) existing output path
Path outputDestination = new Path(args[0] + args[1]);
if (fs.exists(outputDestination)) {
fs.delete(outputDestination, true);
}
// set output path & start job1
FileOutputFormat.setOutputPath(job, outputDestination);
int jobCompletionStatus = job.waitForCompletion(true) ? 0 : 1;
}
private Job configureJob(Path secretsPath, Path saltFilePath, Path inputPath, Path outputPath)
throws Exception {
Job job = Job.getInstance(getConf());
FileInputFormat.setInputPaths(job, inputPath);
FileOutputFormat.setOutputPath(job, outputPath);
job.getConfiguration().set(ObfuscateMapper.SECRET_WORDS_FILE_KEY, secretsPath.toString());
job.getConfiguration().set(ObfuscateMapper.SALT_FILE_KEY, saltFilePath.toString());
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(ObfuscateMapper.class);
job.setNumReduceTasks(0);
job.setJarByClass(getClass());
FileSystem.get(outputPath.toUri(), getConf()).delete(outputPath, true);
return job;
}
private int getNumSecrets(Path secretsPath) throws Exception {
FileSystem fileSystem = FileSystem.get(secretsPath.toUri(), getConf());
FSDataInputStream inputStream = fileSystem.open(secretsPath);
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
String currentLine;
int numLines = 0;
while ((currentLine = reader.readLine()) != null) {
if (!currentLine.isEmpty()) {
numLines++;
}
}
reader.close();
return numLines;
}
private void generateSaltIfNeeded(Path saltFilePath, Path secretsPath) throws Exception {
FileSystem fileSystem = FileSystem.get(saltFilePath.toUri(), getConf());
if (!fileSystem.exists(saltFilePath)) {
FSDataOutputStream outputStream = fileSystem.create(saltFilePath);
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream));
int numSaltsToGenerate = getNumSecrets(secretsPath);
System.out.printf("Generating %d salts\n", numSaltsToGenerate);
for (int i = 0; i < numSaltsToGenerate; i++) {
writer.write(BCrypt.gensalt());
writer.newLine();
}
writer.close();
outputStream.close();
}
}