public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
// Configure intermediate reduces
conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);
// Set reducer class for intermediate reduce
conf.setClass(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduce.class"),
MyGroupByReducer.class,
Reducer.class);
// Set reducer output key class
conf.setClass(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(
1, "mapreduce.map.output.key.class"),
IntWritable.class,
Object.class);
// Set reducer output value class
conf.setClass(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(
1, "mapreduce.map.output.value.class"),
Text.class,
Object.class);
conf.setInt(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduces"), 2);
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: groupbyorderbymrrtest <in> <out>");
System.exit(2);
}
@SuppressWarnings("deprecation")
Job job = new Job(conf, "groupbyorderbymrrtest");
job.setJarByClass(GroupByOrderByMRRTest.class);
// Configure map
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
// Configure reduce
job.setReducerClass(MyOrderByNoOpReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setNumReduceTasks(1);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
TezClient tezClient = new TezClient(new TezConfiguration(conf));
job.submit();
JobID jobId = job.getJobID();
ApplicationId appId = TypeConverter.toYarn(jobId).getAppId();
DAGClient dagClient = tezClient.getDAGClient(appId);
DAGStatus dagStatus = null;
while (true) {
dagStatus = dagClient.getDAGStatus();
if (dagStatus.getState() == DAGStatus.State.RUNNING
|| dagStatus.getState() == DAGStatus.State.SUCCEEDED
|| dagStatus.getState() == DAGStatus.State.FAILED
|| dagStatus.getState() == DAGStatus.State.KILLED
|| dagStatus.getState() == DAGStatus.State.ERROR) {
break;
}
try {
Thread.sleep(500);
} catch (InterruptedException e) {
// continue;
}
}
while (dagStatus.getState() == DAGStatus.State.RUNNING) {
try {
ExampleDriver.printMRRDAGStatus(dagStatus);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// continue;
}
dagStatus = dagClient.getDAGStatus();
} catch (TezException e) {
LOG.fatal("Failed to get application progress. Exiting");
System.exit(-1);
}
}
ExampleDriver.printMRRDAGStatus(dagStatus);
LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
}
