@SuppressWarnings("unchecked")
public void sendTezEventToSourceTasks(TezEvent tezEvent) {
if (!bufferEvents.get()) {
switch (tezEvent.getEventType()) {
case INPUT_READ_ERROR_EVENT:
InputReadErrorEvent event = (InputReadErrorEvent) tezEvent.getEvent();
TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
int destTaskIndex = destAttemptId.getTaskID().getId();
int srcTaskIndex = edgeManager.routeEventToSourceTasks(destTaskIndex, event);
int numConsumers =
edgeManager.getDestinationConsumerTaskNumber(
srcTaskIndex, destinationVertex.getTotalTasks());
TezTaskID srcTaskId = sourceVertex.getTask(srcTaskIndex).getTaskId();
int taskAttemptIndex = event.getVersion();
TezTaskAttemptID srcTaskAttemptId = new TezTaskAttemptID(srcTaskId, taskAttemptIndex);
eventHandler.handle(
new TaskAttemptEventOutputFailed(srcTaskAttemptId, tezEvent, numConsumers));
break;
default:
throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
}
} else {
sourceEventBuffer.add(tezEvent);
}
}
public void testBasicSpeculation(boolean withProgress) throws Exception {
DAG dag = DAG.create("test");
Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
dag.addVertex(vA);
MockTezClient tezClient = createTezSession();
DAGClient dagClient = tezClient.submitDAG(dag);
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
// original attempt is killed and speculative one is successful
TezTaskAttemptID killedTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
TezTaskAttemptID successTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 1);
mockLauncher.updateProgress(withProgress);
// cause speculation trigger
mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
Task task = dagImpl.getTask(killedTaId.getTaskID());
Assert.assertEquals(2, task.getAttempts().size());
Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getID());
TaskAttempt killedAttempt = task.getAttempt(killedTaId);
Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt");
Assert.assertEquals(
TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION,
killedAttempt.getTerminationCause());
if (withProgress) {
// without progress updates occasionally more than 1 task speculates
Assert.assertEquals(
1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
Assert.assertEquals(
1, dagImpl.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
org.apache.tez.dag.app.dag.Vertex v = dagImpl.getVertex(killedTaId.getTaskID().getVertexID());
Assert.assertEquals(
1, v.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
}
tezClient.stop();
}
public void sendTezEventToDestinationTasks(TezEvent tezEvent) {
if (!bufferEvents.get()) {
List<Integer> destTaskIndices = new ArrayList<Integer>();
boolean isDataMovementEvent = true;
switch (tezEvent.getEventType()) {
case INPUT_FAILED_EVENT:
isDataMovementEvent = false;
case DATA_MOVEMENT_EVENT:
Event event = tezEvent.getEvent();
TezTaskAttemptID sourceAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
int sourceTaskIndex = sourceAttemptId.getTaskID().getId();
if (isDataMovementEvent) {
edgeManager.routeEventToDestinationTasks(
(DataMovementEvent) event,
sourceTaskIndex,
destinationVertex.getTotalTasks(),
destTaskIndices);
} else {
edgeManager.routeEventToDestinationTasks(
(InputFailedEvent) event,
sourceTaskIndex,
destinationVertex.getTotalTasks(),
destTaskIndices);
}
for (Integer destTaskIndex : destTaskIndices) {
EventMetaData destMeta =
new EventMetaData(
EventProducerConsumerType.INPUT,
destinationVertex.getName(),
sourceVertex.getName(),
null); // will be filled by Task when sending the event. Is it needed?
if (isDataMovementEvent) {
destMeta.setIndex(((DataMovementEvent) event).getTargetIndex());
} else {
destMeta.setIndex(((InputFailedEvent) event).getTargetIndex());
}
tezEvent.setDestinationInfo(destMeta);
TezTaskID destTaskId = destinationVertex.getTask(destTaskIndex).getTaskId();
sendEventToTask(destTaskId, tezEvent);
}
break;
default:
throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
}
} else {
destinationEventBuffer.add(tezEvent);
}
}