private synchronized void unreserveInternal(Priority priority, FSSchedulerNode node) { Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(priority); RMContainer reservedContainer = reservedContainers.remove(node.getNodeID()); if (reservedContainers.isEmpty()) { this.reservedContainers.remove(priority); } // Reset the re-reservation count resetReReservations(priority); Resource resource = reservedContainer.getContainer().getResource(); Resources.subtractFrom(currentReservation, resource); LOG.info( "Application " + getApplicationId() + " unreserved " + " on node " + node + ", currently has " + reservedContainers.size() + " at priority " + priority + "; currentReservation " + currentReservation); }
public synchronized void unreserveResource(SchedulerApplicationAttempt application) { // adding NP checks as this can now be called for preemption if (reservedContainer != null && reservedContainer.getContainer() != null && reservedContainer.getContainer().getId() != null && reservedContainer.getContainer().getId().getApplicationAttemptId() != null) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = reservedContainer.getContainer().getId().getApplicationAttemptId(); if (!reservedApplication.equals(application.getApplicationAttemptId())) { throw new IllegalStateException( "Trying to unreserve " + " for application " + application.getApplicationAttemptId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } } reservedContainer = null; }
// This is to test container tokens are generated when the containers are // acquired by the AM, not when the containers are allocated @Test public void testContainerTokenGeneratedOnPullRequest() throws Exception { YarnConfiguration conf = new YarnConfiguration(); YarnAPIStorageFactory.setConfiguration(conf); RMStorageFactory.setConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); MockRM rm1 = new MockRM(conf); try { rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); RMApp app1 = rm1.submitApp(200); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>()); ContainerId containerId2 = ContainerId.newInstance(am1.getApplicationAttemptId(), 2); rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED); RMContainer container = rm1.getResourceScheduler().getRMContainer(containerId2); // no container token is generated. Assert.assertEquals(containerId2, container.getContainerId()); Assert.assertNull(container.getContainer().getContainerToken()); // acquire the container. List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()) .getAllocatedContainers(); Assert.assertEquals(containerId2, containers.get(0).getId()); // container token is generated. Assert.assertNotNull(containers.get(0).getContainerToken()); } finally { rm1.stop(); } }
@Override public int compare(RMContainer c1, RMContainer c2) { int ret = c1.getContainer().getPriority().compareTo(c2.getContainer().getPriority()); if (ret == 0) { return c2.getContainerId().compareTo(c1.getContainerId()); } return ret; }
private RMContainer createRMContainer(ApplicationAttemptId appAttId, int id, Resource resource) { ContainerId containerId = ContainerId.newInstance(appAttId, id); RMContainer rmContainer = mock(RMContainer.class); Container container = mock(Container.class); when(container.getResource()).thenReturn(resource); when(container.getNodeId()).thenReturn(nodeId); when(rmContainer.getContainer()).thenReturn(container); when(rmContainer.getContainerId()).thenReturn(containerId); return rmContainer; }
private RMContainer createReservedRMContainer( ApplicationAttemptId appAttId, int id, Resource resource, NodeId nodeId, Priority reservedPriority) { RMContainer container = createRMContainer(appAttId, id, resource); when(container.getReservedResource()).thenReturn(resource); when(container.getReservedPriority()).thenReturn(reservedPriority); when(container.getReservedNode()).thenReturn(nodeId); return container; }
@Test public void testMove() { final String user = "******"; Queue parentQueue = createQueue("parent", null); Queue oldQueue = createQueue("old", parentQueue); Queue newQueue = createQueue("new", parentQueue); QueueMetrics parentMetrics = parentQueue.getMetrics(); QueueMetrics oldMetrics = oldQueue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); ApplicationAttemptId appAttId = createAppAttemptId(0, 0); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt( appAttId, user, oldQueue, oldQueue.getActiveUsersManager(), null); oldMetrics.submitApp(user); // Resource request Resource requestedResource = Resource.newInstance(1536, 2); Priority requestedPriority = Priority.newInstance(2); ResourceRequest request = ResourceRequest.newInstance(requestedPriority, ResourceRequest.ANY, requestedResource, 3); app.updateResourceRequests(Arrays.asList(request)); // Allocated container RMContainer container1 = createRMContainer(appAttId, 1, requestedResource); app.liveContainers.put(container1.getContainerId(), container1); SchedulerNode node = createNode(); app.appSchedulingInfo.allocate( NodeType.OFF_SWITCH, node, requestedPriority, request, container1.getContainer()); // Reserved container Priority prio1 = Priority.newInstance(1); Resource reservedResource = Resource.newInstance(2048, 3); RMContainer container2 = createReservedRMContainer(appAttId, 1, reservedResource, node.getNodeID(), prio1); Map<NodeId, RMContainer> reservations = new HashMap<NodeId, RMContainer>(); reservations.put(node.getNodeID(), container2); app.reservedContainers.put(prio1, reservations); oldMetrics.reserveResource(user, reservedResource); checkQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); checkQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0); checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); app.move(newQueue); checkQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0); checkQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); }
/** * Called when this application already has an existing reservation on the given node. Sees * whether we can turn the reservation into an allocation. Also checks whether the application * needs the reservation anymore, and releases it if not. * * @param node Node that the application has an existing reservation on */ public Resource assignReservedContainer(FSSchedulerNode node) { RMContainer rmContainer = node.getReservedContainer(); Priority priority = rmContainer.getReservedPriority(); // Make sure the application still needs requests at this priority if (getTotalRequiredResources(priority) == 0) { unreserve(priority, node); return Resources.none(); } // Fail early if the reserved container won't fit. // Note that we have an assumption here that there's only one container size // per priority. if (!Resources.fitsIn( node.getReservedContainer().getReservedResource(), node.getAvailableResource())) { return Resources.none(); } return assignContainer(node, true); }
private void updateQueueWithNodeUpdate(NodeUpdateSchedulerEventWrapper eventWrapper) { RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode(); List<UpdatedContainerInfo> containerList = node.getContainerUpdates(); for (UpdatedContainerInfo info : containerList) { for (ContainerStatus status : info.getCompletedContainers()) { ContainerId containerId = status.getContainerId(); SchedulerAppReport app = scheduler.getSchedulerAppInfo(containerId.getApplicationAttemptId()); if (app == null) { // this happens for the AM container // The app have already removed when the NM sends the release // information. continue; } String queue = appQueueMap.get(containerId.getApplicationAttemptId().getApplicationId()); int releasedMemory = 0, releasedVCores = 0; if (status.getExitStatus() == ContainerExitStatus.SUCCESS) { for (RMContainer rmc : app.getLiveContainers()) { if (rmc.getContainerId() == containerId) { releasedMemory += rmc.getContainer().getResource().getMemory(); releasedVCores += rmc.getContainer().getResource().getVirtualCores(); break; } } } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) { if (preemptionContainerMap.containsKey(containerId)) { Resource preResource = preemptionContainerMap.get(containerId); releasedMemory += preResource.getMemory(); releasedVCores += preResource.getVirtualCores(); preemptionContainerMap.remove(containerId); } } // update queue counters updateQueueMetrics(queue, releasedMemory, releasedVCores); } } }
/** * The Scheduler has allocated containers on this node to the given application. * * @param applicationId application * @param rmContainer allocated container */ public synchronized void allocateContainer(ApplicationId applicationId, RMContainer rmContainer) { Container container = rmContainer.getContainer(); deductAvailableResource(container.getResource()); ++numContainers; launchedContainers.put(container.getId(), rmContainer); LOG.info( "Assigned container " + container.getId() + " of capacity " + container.getResource() + " on host " + rmNode.getNodeAddress() + ", which currently has " + numContainers + " containers, " + getUsedResource() + " used and " + getAvailableResource() + " available"); }
public synchronized void containerCompleted( RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) { Container container = rmContainer.getContainer(); ContainerId containerId = container.getId(); // Remove from the list of newly allocated containers if found newlyAllocatedContainers.remove(rmContainer); // Inform the container rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, event)); LOG.info( "Completed container: " + rmContainer.getContainerId() + " in state: " + rmContainer.getState() + " event:" + event); // Remove from the list of containers liveContainers.remove(rmContainer.getContainerId()); RMAuditLogger.logSuccess( getUser(), AuditConstants.RELEASE_CONTAINER, "SchedulerApp", getApplicationId(), containerId); // Update usage metrics Resource containerResource = rmContainer.getContainer().getResource(); queue.getMetrics().releaseResources(getUser(), 1, containerResource); Resources.subtractFrom(currentConsumption, containerResource); // remove from preemption map if it is completed preemptionMap.remove(rmContainer); // Clear resource utilization metrics cache. lastMemoryAggregateAllocationUpdateTime = -1; }
public synchronized RMContainer allocate( NodeType type, FSSchedulerNode node, Priority priority, ResourceRequest request, Container container) { // Update allowed locality level NodeType allowed = allowedLocalityLevel.get(priority); if (allowed != null) { if (allowed.equals(NodeType.OFF_SWITCH) && (type.equals(NodeType.NODE_LOCAL) || type.equals(NodeType.RACK_LOCAL))) { this.resetAllowedLocalityLevel(priority, type); } else if (allowed.equals(NodeType.RACK_LOCAL) && type.equals(NodeType.NODE_LOCAL)) { this.resetAllowedLocalityLevel(priority, type); } } // Required sanity check - AM can call 'allocate' to update resource // request without locking the scheduler, hence we need to check if (getTotalRequiredResources(priority) <= 0) { return null; } // Create RMContainer RMContainer rmContainer = new RMContainerImpl( container, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext); // Add it to allContainers list. newlyAllocatedContainers.add(rmContainer); liveContainers.put(container.getId(), rmContainer); // Update consumption and track allocations List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(type, node, priority, request, container); Resources.addTo(currentConsumption, container.getResource()); // Update resource requests related to "request" and store in RMContainer ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList); // Inform the container rmContainer.handle(new RMContainerEvent(container.getId(), RMContainerEventType.START)); if (LOG.isDebugEnabled()) { LOG.debug( "allocate: applicationAttemptId=" + container.getId().getApplicationAttemptId() + " container=" + container.getId() + " host=" + container.getNodeId().getHost() + " type=" + type); } RMAuditLogger.logSuccess( getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), container.getId()); return rmContainer; }
public synchronized void reserveResource( SchedulerApplicationAttempt application, Priority priority, RMContainer reservedContainer) { // Check if it's already reserved if (this.reservedContainer != null) { // Sanity check if (!reservedContainer.getContainer().getNodeId().equals(getNodeID())) { throw new IllegalStateException( "Trying to reserve" + " container " + reservedContainer + " on node " + reservedContainer.getReservedNode() + " when currently" + " reserved resource " + this.reservedContainer + " on node " + this.reservedContainer.getReservedNode()); } // Cannot reserve more than one application attempt on a given node! // Reservation is still against attempt. if (!this.reservedContainer .getContainer() .getId() .getApplicationAttemptId() .equals(reservedContainer.getContainer().getId().getApplicationAttemptId())) { throw new IllegalStateException( "Trying to reserve" + " container " + reservedContainer + " for application " + application.getApplicationAttemptId() + " when currently" + " reserved container " + this.reservedContainer + " on node " + this); } if (LOG.isDebugEnabled()) { LOG.debug( "Updated reserved container " + reservedContainer.getContainer().getId() + " on node " + this + " for application attempt " + application.getApplicationAttemptId()); } } else { if (LOG.isDebugEnabled()) { LOG.debug( "Reserved container " + reservedContainer.getContainer().getId() + " on node " + this + " for application attempt " + application.getApplicationAttemptId()); } } this.reservedContainer = reservedContainer; }
private void updateQueueWithAllocateRequest( Allocation allocation, ApplicationAttemptId attemptId, List<ResourceRequest> resourceRequests, List<ContainerId> containerIds) throws IOException { // update queue information Resource pendingResource = Resources.createResource(0, 0); Resource allocatedResource = Resources.createResource(0, 0); String queueName = appQueueMap.get(attemptId.getApplicationId()); // container requested for (ResourceRequest request : resourceRequests) { if (request.getResourceName().equals(ResourceRequest.ANY)) { Resources.addTo( pendingResource, Resources.multiply(request.getCapability(), request.getNumContainers())); } } // container allocated for (Container container : allocation.getContainers()) { Resources.addTo(allocatedResource, container.getResource()); Resources.subtractFrom(pendingResource, container.getResource()); } // container released from AM SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId); for (ContainerId containerId : containerIds) { Container container = null; for (RMContainer c : report.getLiveContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { // released allocated containers Resources.subtractFrom(allocatedResource, container.getResource()); } else { for (RMContainer c : report.getReservedContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { // released reserved containers Resources.subtractFrom(pendingResource, container.getResource()); } } } // containers released/preemption from scheduler Set<ContainerId> preemptionContainers = new HashSet<ContainerId>(); if (allocation.getContainerPreemptions() != null) { preemptionContainers.addAll(allocation.getContainerPreemptions()); } if (allocation.getStrictContainerPreemptions() != null) { preemptionContainers.addAll(allocation.getStrictContainerPreemptions()); } if (!preemptionContainers.isEmpty()) { for (ContainerId containerId : preemptionContainers) { if (!preemptionContainerMap.containsKey(containerId)) { Container container = null; for (RMContainer c : report.getLiveContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { preemptionContainerMap.put(containerId, container.getResource()); } } } } // update metrics SortedMap<String, Counter> counterMap = metrics.getCounters(); String names[] = new String[] { "counter.queue." + queueName + ".pending.memory", "counter.queue." + queueName + ".pending.cores", "counter.queue." + queueName + ".allocated.memory", "counter.queue." + queueName + ".allocated.cores" }; int values[] = new int[] { pendingResource.getMemory(), pendingResource.getVirtualCores(), allocatedResource.getMemory(), allocatedResource.getVirtualCores() }; for (int i = names.length - 1; i >= 0; i--) { if (!counterMap.containsKey(names[i])) { metrics.counter(names[i]); counterMap = metrics.getCounters(); } counterMap.get(names[i]).inc(values[i]); } queueLock.lock(); try { if (!schedulerMetrics.isTracked(queueName)) { schedulerMetrics.trackQueue(queueName); } } finally { queueLock.unlock(); } }
/** * Remove the reservation on {@code node} at the given {@link Priority}. This dispatches * SchedulerNode handlers as well. */ public void unreserve(Priority priority, FSSchedulerNode node) { RMContainer rmContainer = node.getReservedContainer(); unreserveInternal(priority, node); node.unreserveResource(this); getMetrics().unreserveResource(getUser(), rmContainer.getContainer().getResource()); }
public void resetPreemptedResources() { preemptedResources = Resources.createResource(0); for (RMContainer container : getPreemptionContainers()) { Resources.addTo(preemptedResources, container.getAllocatedResource()); } }
// related methods public void addPreemption(RMContainer container, long time) { assert preemptionMap.get(container) == null; preemptionMap.put(container, time); Resources.addTo(preemptedResources, container.getAllocatedResource()); }
@Override public void handle(SchedulerEvent schedulerEvent) { // metrics off if (!metricsON) { scheduler.handle(schedulerEvent); return; } if (!running) running = true; // metrics on Timer.Context handlerTimer = null; Timer.Context operationTimer = null; NodeUpdateSchedulerEventWrapper eventWrapper; try { // if (schedulerEvent instanceof NodeUpdateSchedulerEvent) { if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE && schedulerEvent instanceof NodeUpdateSchedulerEvent) { eventWrapper = new NodeUpdateSchedulerEventWrapper((NodeUpdateSchedulerEvent) schedulerEvent); schedulerEvent = eventWrapper; updateQueueWithNodeUpdate(eventWrapper); } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { // check if having AM Container, update resource usage information AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent; ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID(); String queue = appQueueMap.get(appAttemptId.getApplicationId()); SchedulerAppReport app = scheduler.getSchedulerAppInfo(appAttemptId); if (!app.getLiveContainers().isEmpty()) { // have 0 or 1 // should have one container which is AM container RMContainer rmc = app.getLiveContainers().iterator().next(); updateQueueMetrics( queue, rmc.getContainer().getResource().getMemory(), rmc.getContainer().getResource().getVirtualCores()); } } handlerTimer = schedulerHandleTimer.time(); operationTimer = schedulerHandleTimerMap.get(schedulerEvent.getType()).time(); scheduler.handle(schedulerEvent); } finally { if (handlerTimer != null) handlerTimer.stop(); if (operationTimer != null) operationTimer.stop(); schedulerHandleCounter.inc(); schedulerHandleCounterMap.get(schedulerEvent.getType()).inc(); if (schedulerEvent.getType() == SchedulerEventType.APP_REMOVED && schedulerEvent instanceof AppRemovedSchedulerEvent) { SLSRunner.decreaseRemainingApps(); AppRemovedSchedulerEvent appRemoveEvent = (AppRemovedSchedulerEvent) schedulerEvent; appQueueMap.remove(appRemoveEvent.getApplicationID()); } else if (schedulerEvent.getType() == SchedulerEventType.APP_ADDED && schedulerEvent instanceof AppAddedSchedulerEvent) { AppAddedSchedulerEvent appAddEvent = (AppAddedSchedulerEvent) schedulerEvent; String queueName = appAddEvent.getQueue(); appQueueMap.put(appAddEvent.getApplicationId(), queueName); } } }
/** * Remove the reservation on {@code node} at the given {@link Priority}. This dispatches to the * SchedulerApp and SchedulerNode handlers for an unreservation. */ public void unreserve(Priority priority, FSSchedulerNode node) { RMContainer rmContainer = node.getReservedContainer(); app.unreserve(node, priority); node.unreserveResource(app); getMetrics().unreserveResource(app.getUser(), rmContainer.getContainer().getResource()); }
public synchronized void recoverContainer(RMContainer rmContainer) { if (rmContainer.getState().equals(RMContainerState.COMPLETED)) { return; } allocateContainer(rmContainer); }
private Resource assignContainer( FSSchedulerNode node, boolean reserved, TransactionState transactionState) { if (LOG.isDebugEnabled()) { LOG.debug("Node offered to app: " + getName() + " reserved: " + reserved); } if (reserved) { RMContainer rmContainer = node.getReservedContainer(); Priority priority = rmContainer.getReservedPriority(); // Make sure the application still needs requests at this priority if (app.getTotalRequiredResources(priority) == 0) { unreserve(priority, node); return Resources.none(); } } Collection<Priority> prioritiesToTry = (reserved) ? Arrays.asList(node.getReservedContainer().getReservedPriority()) : app.getPriorities(); // For each priority, see if we can schedule a node local, rack local // or off-switch request. Rack of off-switch requests may be delayed // (not scheduled) in order to promote better locality. synchronized (app) { for (Priority priority : prioritiesToTry) { if (app.getTotalRequiredResources(priority) <= 0 || !hasContainerForNode(priority, node)) { continue; } app.addSchedulingOpportunity(priority); ResourceRequest rackLocalRequest = app.getResourceRequest(priority, node.getRackName()); ResourceRequest localRequest = app.getResourceRequest(priority, node.getNodeName()); if (localRequest != null && !localRequest.getRelaxLocality()) { LOG.warn("Relax locality off is not supported on local request: " + localRequest); } NodeType allowedLocality; if (scheduler.isContinuousSchedulingEnabled()) { allowedLocality = app.getAllowedLocalityLevelByTime( priority, scheduler.getNodeLocalityDelayMs(), scheduler.getRackLocalityDelayMs(), scheduler.getClock().getTime()); } else { allowedLocality = app.getAllowedLocalityLevel( priority, scheduler.getNumClusterNodes(), scheduler.getNodeLocalityThreshold(), scheduler.getRackLocalityThreshold()); } if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 && localRequest != null && localRequest.getNumContainers() != 0) { return assignContainer( node, priority, localRequest, NodeType.NODE_LOCAL, reserved, transactionState); } if (rackLocalRequest != null && !rackLocalRequest.getRelaxLocality()) { continue; } if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 && (allowedLocality.equals(NodeType.RACK_LOCAL) || allowedLocality.equals(NodeType.OFF_SWITCH))) { return assignContainer( node, priority, rackLocalRequest, NodeType.RACK_LOCAL, reserved, transactionState); } ResourceRequest offSwitchRequest = app.getResourceRequest(priority, ResourceRequest.ANY); if (offSwitchRequest != null && !offSwitchRequest.getRelaxLocality()) { continue; } if (offSwitchRequest != null && offSwitchRequest.getNumContainers() != 0 && allowedLocality.equals(NodeType.OFF_SWITCH)) { return assignContainer( node, priority, offSwitchRequest, NodeType.OFF_SWITCH, reserved, transactionState); } } } return Resources.none(); }