public static void printDAGStatus(
DAGClient dagClient,
String[] vertexNames,
boolean displayDAGCounters,
boolean displayVertexCounters)
throws IOException, TezException {
Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
DAGStatus dagStatus = dagClient.getDAGStatus((displayDAGCounters ? opts : null));
Progress progress = dagStatus.getDAGProgress();
double vProgressFloat = 0.0f;
if (progress != null) {
System.out.println("");
System.out.println(
"DAG: State: "
+ dagStatus.getState()
+ " Progress: "
+ (progress.getTotalTaskCount() < 0
? formatter.format(0.0f)
: formatter.format(
(double) (progress.getSucceededTaskCount()) / progress.getTotalTaskCount())));
for (String vertexName : vertexNames) {
VertexStatus vStatus =
dagClient.getVertexStatus(vertexName, (displayVertexCounters ? opts : null));
if (vStatus == null) {
System.out.println("Could not retrieve status for vertex: " + vertexName);
continue;
}
Progress vProgress = vStatus.getProgress();
if (vProgress != null) {
vProgressFloat = 0.0f;
if (vProgress.getTotalTaskCount() == 0) {
vProgressFloat = 1.0f;
} else if (vProgress.getTotalTaskCount() > 0) {
vProgressFloat =
(double) vProgress.getSucceededTaskCount() / vProgress.getTotalTaskCount();
}
System.out.println(
"VertexStatus:"
+ " VertexName: "
+ (vertexName.equals("ivertex1") ? "intermediate-reducer" : vertexName)
+ " Progress: "
+ formatter.format(vProgressFloat));
}
if (displayVertexCounters) {
TezCounters counters = vStatus.getVertexCounters();
if (counters != null) {
System.out.println("Vertex Counters for " + vertexName + ": " + counters);
}
}
}
}
if (displayDAGCounters) {
TezCounters counters = dagStatus.getDAGCounters();
if (counters != null) {
System.out.println("DAG Counters: " + counters);
}
}
}
@Test(timeout = 10000)
public void testBasicSpeculationPerVertexConf() throws Exception {
DAG dag = DAG.create("test");
String vNameNoSpec = "A";
String vNameSpec = "B";
Vertex vA = Vertex.create(vNameNoSpec, ProcessorDescriptor.create("Proc.class"), 5);
Vertex vB = Vertex.create(vNameSpec, ProcessorDescriptor.create("Proc.class"), 5);
vA.setConf(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, "false");
dag.addVertex(vA);
dag.addVertex(vB);
// min/max src fraction is set to 1. So vertices will run sequentially
dag.addEdge(
Edge.create(
vA,
vB,
EdgeProperty.create(
DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL,
OutputDescriptor.create("O"),
InputDescriptor.create("I"))));
MockTezClient tezClient = createTezSession();
DAGClient dagClient = tezClient.submitDAG(dag);
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
TezVertexID vertexId = dagImpl.getVertex(vNameSpec).getVertexId();
TezVertexID vertexIdNoSpec = dagImpl.getVertex(vNameNoSpec).getVertexId();
// original attempt is killed and speculative one is successful
TezTaskAttemptID killedTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
TezTaskAttemptID noSpecTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexIdNoSpec, 0), 0);
// cause speculation trigger for both
mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
mockLauncher.setStatusUpdatesForTask(noSpecTaId, 100);
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
org.apache.tez.dag.app.dag.Vertex vSpec = dagImpl.getVertex(vertexId);
org.apache.tez.dag.app.dag.Vertex vNoSpec = dagImpl.getVertex(vertexIdNoSpec);
// speculation for vA but not for vB
Assert.assertTrue(
vSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue() > 0);
Assert.assertEquals(
0, vNoSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
tezClient.stop();
}
public void testBasicSpeculation(boolean withProgress) throws Exception {
DAG dag = DAG.create("test");
Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
dag.addVertex(vA);
MockTezClient tezClient = createTezSession();
DAGClient dagClient = tezClient.submitDAG(dag);
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
// original attempt is killed and speculative one is successful
TezTaskAttemptID killedTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
TezTaskAttemptID successTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 1);
mockLauncher.updateProgress(withProgress);
// cause speculation trigger
mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
Task task = dagImpl.getTask(killedTaId.getTaskID());
Assert.assertEquals(2, task.getAttempts().size());
Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getID());
TaskAttempt killedAttempt = task.getAttempt(killedTaId);
Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt");
Assert.assertEquals(
TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION,
killedAttempt.getTerminationCause());
if (withProgress) {
// without progress updates occasionally more than 1 task speculates
Assert.assertEquals(
1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
Assert.assertEquals(
1, dagImpl.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
org.apache.tez.dag.app.dag.Vertex v = dagImpl.getVertex(killedTaId.getTaskID().getVertexID());
Assert.assertEquals(
1, v.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
}
tezClient.stop();
}
public int run(String[] args) throws Exception {
if (args.length < 1) {
System.err.println(
"MRRSleepJob [-m numMapper] [-r numReducer]"
+ " [-ir numIntermediateReducer]"
+ " [-irs numIntermediateReducerStages]"
+ " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]"
+ " [-irt intermediateReduceSleepTime]"
+ " [-recordt recordSleepTime (msec)]"
+ " [-generateSplitsInAM (false)/true]"
+ " [-writeSplitsToDfs (false)/true]");
ToolRunner.printGenericCommandUsage(System.err);
return 2;
}
int numMapper = 1, numReducer = 1, numIReducer = 1;
long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100, iReduceSleepTime = 1;
int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
int iReduceStagesCount = 1;
boolean writeSplitsToDfs = false;
boolean generateSplitsInAM = false;
boolean splitsOptionFound = false;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-m")) {
numMapper = Integer.parseInt(args[++i]);
} else if (args[i].equals("-r")) {
numReducer = Integer.parseInt(args[++i]);
} else if (args[i].equals("-ir")) {
numIReducer = Integer.parseInt(args[++i]);
} else if (args[i].equals("-mt")) {
mapSleepTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-rt")) {
reduceSleepTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-irt")) {
iReduceSleepTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-irs")) {
iReduceStagesCount = Integer.parseInt(args[++i]);
} else if (args[i].equals("-recordt")) {
recSleepTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-generateSplitsInAM")) {
if (splitsOptionFound) {
throw new RuntimeException(
"Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
}
splitsOptionFound = true;
generateSplitsInAM = Boolean.parseBoolean(args[++i]);
} else if (args[i].equals("-writeSplitsToDfs")) {
if (splitsOptionFound) {
throw new RuntimeException(
"Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
}
splitsOptionFound = true;
writeSplitsToDfs = Boolean.parseBoolean(args[++i]);
}
}
if (numIReducer > 0 && numReducer <= 0) {
throw new RuntimeException("Cannot have intermediate reduces without" + " a final reduce");
}
// sleep for *SleepTime duration in Task by recSleepTime per record
mapSleepCount = (int) Math.ceil(mapSleepTime / ((double) recSleepTime));
reduceSleepCount = (int) Math.ceil(reduceSleepTime / ((double) recSleepTime));
iReduceSleepCount = (int) Math.ceil(iReduceSleepTime / ((double) recSleepTime));
TezConfiguration conf = new TezConfiguration(getConf());
FileSystem remoteFs = FileSystem.get(conf);
conf.set(
TezConfiguration.TEZ_AM_STAGING_DIR,
conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
Path remoteStagingDir =
remoteFs.makeQualified(
new Path(
conf.get(
TezConfiguration.TEZ_AM_STAGING_DIR,
TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
Long.toString(System.currentTimeMillis())));
TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
DAG dag =
createDAG(
remoteFs,
conf,
remoteStagingDir,
numMapper,
numReducer,
iReduceStagesCount,
numIReducer,
mapSleepTime,
mapSleepCount,
reduceSleepTime,
reduceSleepCount,
iReduceSleepTime,
iReduceSleepCount,
writeSplitsToDfs,
generateSplitsInAM);
TezClient tezSession = new TezClient("MRRSleep", conf, false, null, credentials);
tezSession.start();
DAGClient dagClient = tezSession.submitDAG(dag);
while (true) {
DAGStatus status = dagClient.getDAGStatus(null);
LOG.info("DAG Status: " + status);
if (status.isCompleted()) {
break;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// do nothing
}
}
tezSession.stop();
return dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED) ? 0 : 1;
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
// Configure intermediate reduces
conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);
// Set reducer class for intermediate reduce
conf.setClass(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduce.class"),
MyGroupByReducer.class,
Reducer.class);
// Set reducer output key class
conf.setClass(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(
1, "mapreduce.map.output.key.class"),
IntWritable.class,
Object.class);
// Set reducer output value class
conf.setClass(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(
1, "mapreduce.map.output.value.class"),
Text.class,
Object.class);
conf.setInt(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduces"), 2);
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: groupbyorderbymrrtest <in> <out>");
System.exit(2);
}
@SuppressWarnings("deprecation")
Job job = new Job(conf, "groupbyorderbymrrtest");
job.setJarByClass(GroupByOrderByMRRTest.class);
// Configure map
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
// Configure reduce
job.setReducerClass(MyOrderByNoOpReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setNumReduceTasks(1);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
TezClient tezClient = new TezClient(new TezConfiguration(conf));
job.submit();
JobID jobId = job.getJobID();
ApplicationId appId = TypeConverter.toYarn(jobId).getAppId();
DAGClient dagClient = tezClient.getDAGClient(appId);
DAGStatus dagStatus = null;
while (true) {
dagStatus = dagClient.getDAGStatus();
if (dagStatus.getState() == DAGStatus.State.RUNNING
|| dagStatus.getState() == DAGStatus.State.SUCCEEDED
|| dagStatus.getState() == DAGStatus.State.FAILED
|| dagStatus.getState() == DAGStatus.State.KILLED
|| dagStatus.getState() == DAGStatus.State.ERROR) {
break;
}
try {
Thread.sleep(500);
} catch (InterruptedException e) {
// continue;
}
}
while (dagStatus.getState() == DAGStatus.State.RUNNING) {
try {
ExampleDriver.printMRRDAGStatus(dagStatus);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// continue;
}
dagStatus = dagClient.getDAGStatus();
} catch (TezException e) {
LOG.fatal("Failed to get application progress. Exiting");
System.exit(-1);
}
}
ExampleDriver.printMRRDAGStatus(dagStatus);
LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
}