@SuppressWarnings("unchecked") public void sendTezEventToSourceTasks(TezEvent tezEvent) { if (!bufferEvents.get()) { switch (tezEvent.getEventType()) { case INPUT_READ_ERROR_EVENT: InputReadErrorEvent event = (InputReadErrorEvent) tezEvent.getEvent(); TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo().getTaskAttemptID(); int destTaskIndex = destAttemptId.getTaskID().getId(); int srcTaskIndex = edgeManager.routeEventToSourceTasks(destTaskIndex, event); int numConsumers = edgeManager.getDestinationConsumerTaskNumber( srcTaskIndex, destinationVertex.getTotalTasks()); TezTaskID srcTaskId = sourceVertex.getTask(srcTaskIndex).getTaskId(); int taskAttemptIndex = event.getVersion(); TezTaskAttemptID srcTaskAttemptId = new TezTaskAttemptID(srcTaskId, taskAttemptIndex); eventHandler.handle( new TaskAttemptEventOutputFailed(srcTaskAttemptId, tezEvent, numConsumers)); break; default: throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType()); } } else { sourceEventBuffer.add(tezEvent); } }
public void sendTezEventToDestinationTasks(TezEvent tezEvent) { if (!bufferEvents.get()) { List<Integer> destTaskIndices = new ArrayList<Integer>(); boolean isDataMovementEvent = true; switch (tezEvent.getEventType()) { case INPUT_FAILED_EVENT: isDataMovementEvent = false; case DATA_MOVEMENT_EVENT: Event event = tezEvent.getEvent(); TezTaskAttemptID sourceAttemptId = tezEvent.getSourceInfo().getTaskAttemptID(); int sourceTaskIndex = sourceAttemptId.getTaskID().getId(); if (isDataMovementEvent) { edgeManager.routeEventToDestinationTasks( (DataMovementEvent) event, sourceTaskIndex, destinationVertex.getTotalTasks(), destTaskIndices); } else { edgeManager.routeEventToDestinationTasks( (InputFailedEvent) event, sourceTaskIndex, destinationVertex.getTotalTasks(), destTaskIndices); } for (Integer destTaskIndex : destTaskIndices) { EventMetaData destMeta = new EventMetaData( EventProducerConsumerType.INPUT, destinationVertex.getName(), sourceVertex.getName(), null); // will be filled by Task when sending the event. Is it needed? if (isDataMovementEvent) { destMeta.setIndex(((DataMovementEvent) event).getTargetIndex()); } else { destMeta.setIndex(((InputFailedEvent) event).getTargetIndex()); } tezEvent.setDestinationInfo(destMeta); TezTaskID destTaskId = destinationVertex.getTask(destTaskIndex).getTaskId(); sendEventToTask(destTaskId, tezEvent); } break; default: throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType()); } } else { destinationEventBuffer.add(tezEvent); } }
@Override public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException, TezException { // Keep-alive information. The client should be informed and will have to take care of // re-submitting the work. // Some parts of fault tolerance go here. // This also provides completion information, and a possible notification when task actually // starts running (first heartbeat) if (LOG.isDebugEnabled()) { LOG.debug("Received heartbeat from container, request=" + request); } // Incoming events can be ignored until the point when shuffle needs to be handled, instead of // just scans. TezHeartbeatResponse response = new TezHeartbeatResponse(); response.setLastRequestId(request.getRequestId()); // Assuming TaskAttemptId and FragmentIdentifierString are the same. Verify this. TezTaskAttemptID taskAttemptId = request.getCurrentTaskAttemptID(); String taskAttemptIdString = taskAttemptId.toString(); updateHeartbeatInfo(taskAttemptIdString); List<TezEvent> tezEvents = null; PendingEventData pendingEventData = pendingEvents.remove(taskAttemptIdString); if (pendingEventData == null) { tezEvents = Collections.emptyList(); // If this heartbeat was not from a pending event and it's not in our list of registered // tasks, if (!registeredTasks.containsKey(taskAttemptIdString)) { LOG.info("Unexpected heartbeat from " + taskAttemptIdString); response.setShouldDie(); // Do any of the other fields need to be set? return response; } } else { tezEvents = pendingEventData.tezEvents; // Tasks removed from the pending list should then be added to the registered list. registeredTasks.put(taskAttemptIdString, pendingEventData.heartbeatInfo); } response.setLastRequestId(request.getRequestId()); // Irrelevant from eventIds. This can be tracked in the AM itself, instead of polluting the // task. // Also since we have all the MRInput events here - they'll all be sent in together. response.setNextFromEventId(0); // Irrelevant. See comment above. response.setNextPreRoutedEventId(0); // Irrelevant. See comment above. response.setEvents(tezEvents); List<TezEvent> inEvents = request.getEvents(); if (LOG.isDebugEnabled()) { LOG.debug( "Heartbeat from " + taskAttemptIdString + " events: " + (inEvents != null ? inEvents.size() : -1)); } for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) { EventType eventType = tezEvent.getEventType(); switch (eventType) { case TASK_ATTEMPT_COMPLETED_EVENT: LOG.debug("Task completed event for " + taskAttemptIdString); registeredTasks.remove(taskAttemptIdString); break; case TASK_ATTEMPT_FAILED_EVENT: LOG.debug("Task failed event for " + taskAttemptIdString); registeredTasks.remove(taskAttemptIdString); break; case TASK_STATUS_UPDATE_EVENT: // If we want to handle counters LOG.debug("Task update event for " + taskAttemptIdString); break; default: LOG.warn("Unhandled event type " + eventType); break; } } // Pass the request on to the responder try { if (responder != null) { responder.heartbeat(request); } } catch (Exception err) { LOG.error("Error during responder execution", err); } return response; }