private void updateQueueWithNodeUpdate(NodeUpdateSchedulerEventWrapper eventWrapper) { RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode(); List<UpdatedContainerInfo> containerList = node.getContainerUpdates(); for (UpdatedContainerInfo info : containerList) { for (ContainerStatus status : info.getCompletedContainers()) { ContainerId containerId = status.getContainerId(); SchedulerAppReport app = scheduler.getSchedulerAppInfo(containerId.getApplicationAttemptId()); if (app == null) { // this happens for the AM container // The app have already removed when the NM sends the release // information. continue; } String queue = appQueueMap.get(containerId.getApplicationAttemptId().getApplicationId()); int releasedMemory = 0, releasedVCores = 0; if (status.getExitStatus() == ContainerExitStatus.SUCCESS) { for (RMContainer rmc : app.getLiveContainers()) { if (rmc.getContainerId() == containerId) { releasedMemory += rmc.getContainer().getResource().getMemory(); releasedVCores += rmc.getContainer().getResource().getVirtualCores(); break; } } } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) { if (preemptionContainerMap.containsKey(containerId)) { Resource preResource = preemptionContainerMap.get(containerId); releasedMemory += preResource.getMemory(); releasedVCores += preResource.getVirtualCores(); preemptionContainerMap.remove(containerId); } } // update queue counters updateQueueMetrics(queue, releasedMemory, releasedVCores); } } }
private void updateQueueWithAllocateRequest( Allocation allocation, ApplicationAttemptId attemptId, List<ResourceRequest> resourceRequests, List<ContainerId> containerIds) throws IOException { // update queue information Resource pendingResource = Resources.createResource(0, 0); Resource allocatedResource = Resources.createResource(0, 0); String queueName = appQueueMap.get(attemptId.getApplicationId()); // container requested for (ResourceRequest request : resourceRequests) { if (request.getResourceName().equals(ResourceRequest.ANY)) { Resources.addTo( pendingResource, Resources.multiply(request.getCapability(), request.getNumContainers())); } } // container allocated for (Container container : allocation.getContainers()) { Resources.addTo(allocatedResource, container.getResource()); Resources.subtractFrom(pendingResource, container.getResource()); } // container released from AM SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId); for (ContainerId containerId : containerIds) { Container container = null; for (RMContainer c : report.getLiveContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { // released allocated containers Resources.subtractFrom(allocatedResource, container.getResource()); } else { for (RMContainer c : report.getReservedContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { // released reserved containers Resources.subtractFrom(pendingResource, container.getResource()); } } } // containers released/preemption from scheduler Set<ContainerId> preemptionContainers = new HashSet<ContainerId>(); if (allocation.getContainerPreemptions() != null) { preemptionContainers.addAll(allocation.getContainerPreemptions()); } if (allocation.getStrictContainerPreemptions() != null) { preemptionContainers.addAll(allocation.getStrictContainerPreemptions()); } if (!preemptionContainers.isEmpty()) { for (ContainerId containerId : preemptionContainers) { if (!preemptionContainerMap.containsKey(containerId)) { Container container = null; for (RMContainer c : report.getLiveContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { preemptionContainerMap.put(containerId, container.getResource()); } } } } // update metrics SortedMap<String, Counter> counterMap = metrics.getCounters(); String names[] = new String[] { "counter.queue." + queueName + ".pending.memory", "counter.queue." + queueName + ".pending.cores", "counter.queue." + queueName + ".allocated.memory", "counter.queue." + queueName + ".allocated.cores" }; int values[] = new int[] { pendingResource.getMemory(), pendingResource.getVirtualCores(), allocatedResource.getMemory(), allocatedResource.getVirtualCores() }; for (int i = names.length - 1; i >= 0; i--) { if (!counterMap.containsKey(names[i])) { metrics.counter(names[i]); counterMap = metrics.getCounters(); } counterMap.get(names[i]).inc(values[i]); } queueLock.lock(); try { if (!schedulerMetrics.isTracked(queueName)) { schedulerMetrics.trackQueue(queueName); } } finally { queueLock.unlock(); } }
@Override public void handle(SchedulerEvent schedulerEvent) { // metrics off if (!metricsON) { scheduler.handle(schedulerEvent); return; } if (!running) running = true; // metrics on Timer.Context handlerTimer = null; Timer.Context operationTimer = null; NodeUpdateSchedulerEventWrapper eventWrapper; try { // if (schedulerEvent instanceof NodeUpdateSchedulerEvent) { if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE && schedulerEvent instanceof NodeUpdateSchedulerEvent) { eventWrapper = new NodeUpdateSchedulerEventWrapper((NodeUpdateSchedulerEvent) schedulerEvent); schedulerEvent = eventWrapper; updateQueueWithNodeUpdate(eventWrapper); } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { // check if having AM Container, update resource usage information AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent; ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID(); String queue = appQueueMap.get(appAttemptId.getApplicationId()); SchedulerAppReport app = scheduler.getSchedulerAppInfo(appAttemptId); if (!app.getLiveContainers().isEmpty()) { // have 0 or 1 // should have one container which is AM container RMContainer rmc = app.getLiveContainers().iterator().next(); updateQueueMetrics( queue, rmc.getContainer().getResource().getMemory(), rmc.getContainer().getResource().getVirtualCores()); } } handlerTimer = schedulerHandleTimer.time(); operationTimer = schedulerHandleTimerMap.get(schedulerEvent.getType()).time(); scheduler.handle(schedulerEvent); } finally { if (handlerTimer != null) handlerTimer.stop(); if (operationTimer != null) operationTimer.stop(); schedulerHandleCounter.inc(); schedulerHandleCounterMap.get(schedulerEvent.getType()).inc(); if (schedulerEvent.getType() == SchedulerEventType.APP_REMOVED && schedulerEvent instanceof AppRemovedSchedulerEvent) { SLSRunner.decreaseRemainingApps(); AppRemovedSchedulerEvent appRemoveEvent = (AppRemovedSchedulerEvent) schedulerEvent; appQueueMap.remove(appRemoveEvent.getApplicationID()); } else if (schedulerEvent.getType() == SchedulerEventType.APP_ADDED && schedulerEvent instanceof AppAddedSchedulerEvent) { AppAddedSchedulerEvent appAddEvent = (AppAddedSchedulerEvent) schedulerEvent; String queueName = appAddEvent.getQueue(); appQueueMap.put(appAddEvent.getApplicationId(), queueName); } } }