public static void main(String[] args) throws Exception {
final String NAME_NODE = "hdfs://sandbox.hortonworks.com:8020";
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(WordCount.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(NullWritable.class);
if (args.length > 2) {
job.setNumReduceTasks(Integer.parseInt(args[2]));
}
job.setMapperClass(CountMapper.class);
job.setReducerClass(CountReducer.class);
job.setJarByClass(WordCount.class);
job.setNumReduceTasks(1);
FileInputFormat.addInputPath(job, new Path(args[0] + "data/plot_summaries.txt"));
FileSystem fs = FileSystem.get(conf);
// handle (e.g. delete) existing output path
Path outputDestination = new Path(args[0] + args[1]);
if (fs.exists(outputDestination)) {
fs.delete(outputDestination, true);
}
// set output path & start job1
FileOutputFormat.setOutputPath(job, outputDestination);
int jobCompletionStatus = job.waitForCompletion(true) ? 0 : 1;
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] remainArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (remainArgs.length != 2) {
System.err.println("Usage: wordcount <input> <output>");
System.exit(1);
}
Job job = new Job(conf, "wordcount");
job.setJarByClass(WordCount.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(Map.class);
job.setCombinerClass(Reduce.class);
job.setReducerClass(Reduce.class);
job.setNumReduceTasks(4);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileSystem.get(conf).delete(new Path(remainArgs[1]), true);
FileInputFormat.setInputPaths(job, new Path(remainArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(remainArgs[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
public static void main(String[] args) throws Exception {
Job job = new Job();
job.setJarByClass(Sort.class);
job.setJobName("Sort");
FileInputFormat.addInputPath(job, new Path("hdfs://localhost:9000/input/"));
FileOutputFormat.setOutputPath(job, new Path("hdfs://localhost:9000/output/"));
job.setMapperClass(Map.class);
// job.setCombinerClass(Reduce.class);
job.setReducerClass(Reduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setNumReduceTasks(2);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
public int run(String[] args) throws Exception {
Job job = new Job(getConf());
job.setJarByClass(HadoopNBFilter.class);
job.setJobName("hadoopnbfilter");
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setNumReduceTasks(Integer.parseInt(args[2]));
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
boolean jobCompleted = job.waitForCompletion(true);
return jobCompleted ? 0 : 1;
}
public X(String input, String output, Configuration config, FileSystem hdfs) throws Exception {
Job job = new Job(config);
job.setJarByClass(codesquare.badges.badge_21_22_23.Pass2.class);
job.setJobName("");
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
job.setNumReduceTasks(1);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path("/user/interns/Extras/bossList.txt"));
FileOutputFormat.setOutputPath(job, new Path(output));
try {
job.waitForCompletion(true);
} catch (IOException e) {
System.out.println("No Input Paths to run this MapReduce on!");
}
}
public static void runJob(String mysqlJar, String output) throws Exception {
Configuration conf = new Configuration();
JobHelper.addJarForJob(conf, mysqlJar);
DBConfiguration.configureDB(
conf,
"com.mysql.jdbc.Driver",
"jdbc:mysql://localhost/sqoop_test" + "?user=hip_sqoop_user&password=password");
Job job = new Job(conf);
job.setJarByClass(DBImportExportMapReduce.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
job.setInputFormatClass(DBInputFormat.class);
job.setOutputFormatClass(DBOutputFormat.class);
job.setMapOutputKeyClass(StockRecord.class);
job.setMapOutputValueClass(NullWritable.class);
job.setOutputKeyClass(StockRecord.class);
job.setOutputValueClass(NullWritable.class);
job.getConfiguration().setInt("mapred.map.tasks", 4);
job.setNumReduceTasks(4);
DBInputFormat.setInput(
job, StockRecord.class, "select * from stocks", "SELECT COUNT(id) FROM stocks");
DBOutputFormat.setOutput(job, "stocks_export", StockRecord.fields);
Path outputPath = new Path(output);
FileOutputFormat.setOutputPath(job, outputPath);
outputPath.getFileSystem(conf).delete(outputPath, true);
job.waitForCompletion(true);
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = new Job(conf, "wordcount");
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(TopMapper.class);
job.setReducerClass(TopReducer.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setNumReduceTasks(1);
job.setJarByClass(WordCount_e3.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}