Esempio n. 1
0
 @SuppressWarnings("unchecked")
 public void sendTezEventToSourceTasks(TezEvent tezEvent) {
   if (!bufferEvents.get()) {
     switch (tezEvent.getEventType()) {
       case INPUT_READ_ERROR_EVENT:
         InputReadErrorEvent event = (InputReadErrorEvent) tezEvent.getEvent();
         TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
         int destTaskIndex = destAttemptId.getTaskID().getId();
         int srcTaskIndex = edgeManager.routeEventToSourceTasks(destTaskIndex, event);
         int numConsumers =
             edgeManager.getDestinationConsumerTaskNumber(
                 srcTaskIndex, destinationVertex.getTotalTasks());
         TezTaskID srcTaskId = sourceVertex.getTask(srcTaskIndex).getTaskId();
         int taskAttemptIndex = event.getVersion();
         TezTaskAttemptID srcTaskAttemptId = new TezTaskAttemptID(srcTaskId, taskAttemptIndex);
         eventHandler.handle(
             new TaskAttemptEventOutputFailed(srcTaskAttemptId, tezEvent, numConsumers));
         break;
       default:
         throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
     }
   } else {
     sourceEventBuffer.add(tezEvent);
   }
 }
Esempio n. 2
0
  public void testBasicSpeculation(boolean withProgress) throws Exception {
    DAG dag = DAG.create("test");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
    dag.addVertex(vA);

    MockTezClient tezClient = createTezSession();

    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
    // original attempt is killed and speculative one is successful
    TezTaskAttemptID killedTaId =
        TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TezTaskAttemptID successTaId =
        TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 1);

    mockLauncher.updateProgress(withProgress);
    // cause speculation trigger
    mockLauncher.setStatusUpdatesForTask(killedTaId, 100);

    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    Task task = dagImpl.getTask(killedTaId.getTaskID());
    Assert.assertEquals(2, task.getAttempts().size());
    Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getID());
    TaskAttempt killedAttempt = task.getAttempt(killedTaId);
    Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt");
    Assert.assertEquals(
        TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION,
        killedAttempt.getTerminationCause());
    if (withProgress) {
      // without progress updates occasionally more than 1 task speculates
      Assert.assertEquals(
          1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
      Assert.assertEquals(
          1, dagImpl.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
      org.apache.tez.dag.app.dag.Vertex v = dagImpl.getVertex(killedTaId.getTaskID().getVertexID());
      Assert.assertEquals(
          1, v.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
    }
    tezClient.stop();
  }
Esempio n. 3
0
 public void sendTezEventToDestinationTasks(TezEvent tezEvent) {
   if (!bufferEvents.get()) {
     List<Integer> destTaskIndices = new ArrayList<Integer>();
     boolean isDataMovementEvent = true;
     switch (tezEvent.getEventType()) {
       case INPUT_FAILED_EVENT:
         isDataMovementEvent = false;
       case DATA_MOVEMENT_EVENT:
         Event event = tezEvent.getEvent();
         TezTaskAttemptID sourceAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
         int sourceTaskIndex = sourceAttemptId.getTaskID().getId();
         if (isDataMovementEvent) {
           edgeManager.routeEventToDestinationTasks(
               (DataMovementEvent) event,
               sourceTaskIndex,
               destinationVertex.getTotalTasks(),
               destTaskIndices);
         } else {
           edgeManager.routeEventToDestinationTasks(
               (InputFailedEvent) event,
               sourceTaskIndex,
               destinationVertex.getTotalTasks(),
               destTaskIndices);
         }
         for (Integer destTaskIndex : destTaskIndices) {
           EventMetaData destMeta =
               new EventMetaData(
                   EventProducerConsumerType.INPUT,
                   destinationVertex.getName(),
                   sourceVertex.getName(),
                   null); // will be filled by Task when sending the event. Is it needed?
           if (isDataMovementEvent) {
             destMeta.setIndex(((DataMovementEvent) event).getTargetIndex());
           } else {
             destMeta.setIndex(((InputFailedEvent) event).getTargetIndex());
           }
           tezEvent.setDestinationInfo(destMeta);
           TezTaskID destTaskId = destinationVertex.getTask(destTaskIndex).getTaskId();
           sendEventToTask(destTaskId, tezEvent);
         }
         break;
       default:
         throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
     }
   } else {
     destinationEventBuffer.add(tezEvent);
   }
 }