/** Tracks the given new runnable app for purposes of maintaining max running app limits. */ public void trackRunnableApp(FSAppAttempt app) { String user = app.getUser(); FSLeafQueue queue = app.getQueue(); // Increment running counts for all parent queues FSParentQueue parent = queue.getParent(); while (parent != null) { parent.incrementRunnableApps(); parent = parent.getParent(); } Integer userNumRunnable = usersNumRunnableApps.get(user); usersNumRunnableApps.put(user, (userNumRunnable == null ? 0 : userNumRunnable) + 1); }
@Test public void testUpdateDemand() { AppSchedulable app = mock(AppSchedulable.class); Mockito.when(app.getDemand()).thenReturn(maxResource); schedulable.addAppSchedulable(app); schedulable.addAppSchedulable(app); schedulable.updateDemand(); assertTrue( "Demand is greater than max allowed ", Resources.equals(schedulable.getDemand(), maxResource)); }
/** * Checks to see whether any other applications runnable now that the given application has been * removed from the given queue. And makes them so. * * <p>Runs in O(n log(n)) where n is the number of queues that are under the highest queue that * went from having no slack to having slack. */ public void updateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); // childqueueX might have no pending apps itself, but if a queue higher up // in the hierarchy parentqueueY has a maxRunningApps set, an app completion // in childqueueX could allow an app in some other distant child of // parentqueueY to become runnable. // An app removal will only possibly allow another app to become runnable if // the queue was already at its max before the removal. // Thus we find the ancestor queue highest in the tree for which the app // that was at its maxRunningApps before the removal. FSQueue highestQueueWithAppsNowRunnable = (queue.getNumRunnableApps() == allocConf.getQueueMaxApps(queue.getName()) - 1) ? queue : null; FSParentQueue parent = queue.getParent(); while (parent != null) { if (parent.getNumRunnableApps() == allocConf.getQueueMaxApps(parent.getName()) - 1) { highestQueueWithAppsNowRunnable = parent; } parent = parent.getParent(); } List<List<FSAppAttempt>> appsNowMaybeRunnable = new ArrayList<List<FSAppAttempt>>(); // Compile lists of apps which may now be runnable // We gather lists instead of building a set of all non-runnable apps so // that this whole operation can be O(number of queues) instead of // O(number of apps) if (highestQueueWithAppsNowRunnable != null) { gatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable); } String user = app.getUser(); Integer userNumRunning = usersNumRunnableApps.get(user); if (userNumRunning == null) { userNumRunning = 0; } if (userNumRunning == allocConf.getUserMaxApps(user) - 1) { List<FSAppAttempt> userWaitingApps = usersNonRunnableApps.get(user); if (userWaitingApps != null) { appsNowMaybeRunnable.add(userWaitingApps); } } updateAppsRunnability(appsNowMaybeRunnable, appsNowMaybeRunnable.size()); }
/** * Updates the relevant tracking variables after a runnable app with the given queue and user has * been removed. */ public void untrackRunnableApp(FSAppAttempt app) { // Update usersRunnableApps String user = app.getUser(); int newUserNumRunning = usersNumRunnableApps.get(user) - 1; if (newUserNumRunning == 0) { usersNumRunnableApps.remove(user); } else { usersNumRunnableApps.put(user, newUserNumRunning); } // Update runnable app bookkeeping for queues FSLeafQueue queue = app.getQueue(); FSParentQueue parent = queue.getParent(); while (parent != null) { parent.decrementRunnableApps(); parent = parent.getParent(); } }
private FSAppAttempt addApp(FSLeafQueue queue, String user) { ApplicationId appId = ApplicationId.newInstance(0l, appNum++); ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 0); boolean runnable = maxAppsEnforcer.canAppBeRunnable(queue, user); FSAppAttempt app = new FSAppAttempt(scheduler, attId, user, queue, null, rmContext); queue.addApp(app, runnable); if (runnable) { maxAppsEnforcer.trackRunnableApp(app); } else { maxAppsEnforcer.trackNonRunnableApp(app); } return app; }
@Test public void testRemoveEnablingOrderedByStartTime() { FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true); queueMaxApps.put("root.queue1", 2); FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); clock.tick(20); addApp(leaf1, "user"); assertEquals(1, leaf1.getRunnableAppSchedulables().size()); assertEquals(1, leaf2.getRunnableAppSchedulables().size()); assertEquals(1, leaf1.getNonRunnableAppSchedulables().size()); assertEquals(1, leaf2.getNonRunnableAppSchedulables().size()); removeApp(app1); assertEquals(0, leaf1.getRunnableAppSchedulables().size()); assertEquals(2, leaf2.getRunnableAppSchedulables().size()); assertEquals(0, leaf2.getNonRunnableAppSchedulables().size()); }
@Test public void testRemoveEnablesOneByQueueOneByUser() { FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.leaf2", true); queueMaxApps.put("root.queue1.leaf1", 2); userMaxApps.put("user1", 1); FSAppAttempt app1 = addApp(leaf1, "user1"); addApp(leaf1, "user2"); addApp(leaf1, "user3"); addApp(leaf2, "user1"); assertEquals(2, leaf1.getRunnableAppSchedulables().size()); assertEquals(1, leaf1.getNonRunnableAppSchedulables().size()); assertEquals(1, leaf2.getNonRunnableAppSchedulables().size()); removeApp(app1); assertEquals(2, leaf1.getRunnableAppSchedulables().size()); assertEquals(1, leaf2.getRunnableAppSchedulables().size()); assertEquals(0, leaf1.getNonRunnableAppSchedulables().size()); assertEquals(0, leaf2.getNonRunnableAppSchedulables().size()); }
@Test public void testMultipleAppsWaitingOnCousinQueue() { FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true); queueMaxApps.put("root.queue1", 2); FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); assertEquals(1, leaf1.getRunnableAppSchedulables().size()); assertEquals(1, leaf2.getRunnableAppSchedulables().size()); assertEquals(2, leaf2.getNonRunnableAppSchedulables().size()); removeApp(app1); assertEquals(0, leaf1.getRunnableAppSchedulables().size()); assertEquals(2, leaf2.getRunnableAppSchedulables().size()); assertEquals(1, leaf2.getNonRunnableAppSchedulables().size()); }
@Test public void testRemoveDoesNotEnableAnyApp() { FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue2", true); queueMaxApps.put("root", 2); queueMaxApps.put("root.queue1", 1); queueMaxApps.put("root.queue2", 1); FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); assertEquals(1, leaf1.getRunnableAppSchedulables().size()); assertEquals(1, leaf2.getRunnableAppSchedulables().size()); assertEquals(1, leaf2.getNonRunnableAppSchedulables().size()); removeApp(app1); assertEquals(0, leaf1.getRunnableAppSchedulables().size()); assertEquals(1, leaf2.getRunnableAppSchedulables().size()); assertEquals(1, leaf2.getNonRunnableAppSchedulables().size()); }
/** Get metrics reference from containing queue. */ public QueueMetrics getMetrics() { return queue.getMetrics(); }