Esempio n. 1
0
 @SuppressWarnings("unchecked")
 public void sendTezEventToSourceTasks(TezEvent tezEvent) {
   if (!bufferEvents.get()) {
     switch (tezEvent.getEventType()) {
       case INPUT_READ_ERROR_EVENT:
         InputReadErrorEvent event = (InputReadErrorEvent) tezEvent.getEvent();
         TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
         int destTaskIndex = destAttemptId.getTaskID().getId();
         int srcTaskIndex = edgeManager.routeEventToSourceTasks(destTaskIndex, event);
         int numConsumers =
             edgeManager.getDestinationConsumerTaskNumber(
                 srcTaskIndex, destinationVertex.getTotalTasks());
         TezTaskID srcTaskId = sourceVertex.getTask(srcTaskIndex).getTaskId();
         int taskAttemptIndex = event.getVersion();
         TezTaskAttemptID srcTaskAttemptId = new TezTaskAttemptID(srcTaskId, taskAttemptIndex);
         eventHandler.handle(
             new TaskAttemptEventOutputFailed(srcTaskAttemptId, tezEvent, numConsumers));
         break;
       default:
         throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
     }
   } else {
     sourceEventBuffer.add(tezEvent);
   }
 }
Esempio n. 2
0
 public void sendTezEventToDestinationTasks(TezEvent tezEvent) {
   if (!bufferEvents.get()) {
     List<Integer> destTaskIndices = new ArrayList<Integer>();
     boolean isDataMovementEvent = true;
     switch (tezEvent.getEventType()) {
       case INPUT_FAILED_EVENT:
         isDataMovementEvent = false;
       case DATA_MOVEMENT_EVENT:
         Event event = tezEvent.getEvent();
         TezTaskAttemptID sourceAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
         int sourceTaskIndex = sourceAttemptId.getTaskID().getId();
         if (isDataMovementEvent) {
           edgeManager.routeEventToDestinationTasks(
               (DataMovementEvent) event,
               sourceTaskIndex,
               destinationVertex.getTotalTasks(),
               destTaskIndices);
         } else {
           edgeManager.routeEventToDestinationTasks(
               (InputFailedEvent) event,
               sourceTaskIndex,
               destinationVertex.getTotalTasks(),
               destTaskIndices);
         }
         for (Integer destTaskIndex : destTaskIndices) {
           EventMetaData destMeta =
               new EventMetaData(
                   EventProducerConsumerType.INPUT,
                   destinationVertex.getName(),
                   sourceVertex.getName(),
                   null); // will be filled by Task when sending the event. Is it needed?
           if (isDataMovementEvent) {
             destMeta.setIndex(((DataMovementEvent) event).getTargetIndex());
           } else {
             destMeta.setIndex(((InputFailedEvent) event).getTargetIndex());
           }
           tezEvent.setDestinationInfo(destMeta);
           TezTaskID destTaskId = destinationVertex.getTask(destTaskIndex).getTaskId();
           sendEventToTask(destTaskId, tezEvent);
         }
         break;
       default:
         throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
     }
   } else {
     destinationEventBuffer.add(tezEvent);
   }
 }
    @Override
    public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request)
        throws IOException, TezException {
      // Keep-alive information. The client should be informed and will have to take care of
      // re-submitting the work.
      // Some parts of fault tolerance go here.

      // This also provides completion information, and a possible notification when task actually
      // starts running (first heartbeat)

      if (LOG.isDebugEnabled()) {
        LOG.debug("Received heartbeat from container, request=" + request);
      }

      // Incoming events can be ignored until the point when shuffle needs to be handled, instead of
      // just scans.
      TezHeartbeatResponse response = new TezHeartbeatResponse();

      response.setLastRequestId(request.getRequestId());
      // Assuming TaskAttemptId and FragmentIdentifierString are the same. Verify this.
      TezTaskAttemptID taskAttemptId = request.getCurrentTaskAttemptID();
      String taskAttemptIdString = taskAttemptId.toString();

      updateHeartbeatInfo(taskAttemptIdString);

      List<TezEvent> tezEvents = null;
      PendingEventData pendingEventData = pendingEvents.remove(taskAttemptIdString);
      if (pendingEventData == null) {
        tezEvents = Collections.emptyList();

        // If this heartbeat was not from a pending event and it's not in our list of registered
        // tasks,
        if (!registeredTasks.containsKey(taskAttemptIdString)) {
          LOG.info("Unexpected heartbeat from " + taskAttemptIdString);
          response.setShouldDie(); // Do any of the other fields need to be set?
          return response;
        }
      } else {
        tezEvents = pendingEventData.tezEvents;
        // Tasks removed from the pending list should then be added to the registered list.
        registeredTasks.put(taskAttemptIdString, pendingEventData.heartbeatInfo);
      }

      response.setLastRequestId(request.getRequestId());
      // Irrelevant from eventIds. This can be tracked in the AM itself, instead of polluting the
      // task.
      // Also since we have all the MRInput events here - they'll all be sent in together.
      response.setNextFromEventId(0); // Irrelevant. See comment above.
      response.setNextPreRoutedEventId(0); // Irrelevant. See comment above.
      response.setEvents(tezEvents);

      List<TezEvent> inEvents = request.getEvents();
      if (LOG.isDebugEnabled()) {
        LOG.debug(
            "Heartbeat from "
                + taskAttemptIdString
                + " events: "
                + (inEvents != null ? inEvents.size() : -1));
      }
      for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) {
        EventType eventType = tezEvent.getEventType();
        switch (eventType) {
          case TASK_ATTEMPT_COMPLETED_EVENT:
            LOG.debug("Task completed event for " + taskAttemptIdString);
            registeredTasks.remove(taskAttemptIdString);
            break;
          case TASK_ATTEMPT_FAILED_EVENT:
            LOG.debug("Task failed event for " + taskAttemptIdString);
            registeredTasks.remove(taskAttemptIdString);
            break;
          case TASK_STATUS_UPDATE_EVENT:
            // If we want to handle counters
            LOG.debug("Task update event for " + taskAttemptIdString);
            break;
          default:
            LOG.warn("Unhandled event type " + eventType);
            break;
        }
      }

      // Pass the request on to the responder
      try {
        if (responder != null) {
          responder.heartbeat(request);
        }
      } catch (Exception err) {
        LOG.error("Error during responder execution", err);
      }

      return response;
    }