/** Tracks the given new runnable app for purposes of maintaining max running app limits. */
  public void trackRunnableApp(FSAppAttempt app) {
    String user = app.getUser();
    FSLeafQueue queue = app.getQueue();
    // Increment running counts for all parent queues
    FSParentQueue parent = queue.getParent();
    while (parent != null) {
      parent.incrementRunnableApps();
      parent = parent.getParent();
    }

    Integer userNumRunnable = usersNumRunnableApps.get(user);
    usersNumRunnableApps.put(user, (userNumRunnable == null ? 0 : userNumRunnable) + 1);
  }
Example #2
0
  @Test
  public void testUpdateDemand() {
    AppSchedulable app = mock(AppSchedulable.class);
    Mockito.when(app.getDemand()).thenReturn(maxResource);

    schedulable.addAppSchedulable(app);
    schedulable.addAppSchedulable(app);

    schedulable.updateDemand();

    assertTrue(
        "Demand is greater than max allowed ",
        Resources.equals(schedulable.getDemand(), maxResource));
  }
  /**
   * Checks to see whether any other applications runnable now that the given application has been
   * removed from the given queue. And makes them so.
   *
   * <p>Runs in O(n log(n)) where n is the number of queues that are under the highest queue that
   * went from having no slack to having slack.
   */
  public void updateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue) {
    AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();

    // childqueueX might have no pending apps itself, but if a queue higher up
    // in the hierarchy parentqueueY has a maxRunningApps set, an app completion
    // in childqueueX could allow an app in some other distant child of
    // parentqueueY to become runnable.
    // An app removal will only possibly allow another app to become runnable if
    // the queue was already at its max before the removal.
    // Thus we find the ancestor queue highest in the tree for which the app
    // that was at its maxRunningApps before the removal.
    FSQueue highestQueueWithAppsNowRunnable =
        (queue.getNumRunnableApps() == allocConf.getQueueMaxApps(queue.getName()) - 1)
            ? queue
            : null;
    FSParentQueue parent = queue.getParent();
    while (parent != null) {
      if (parent.getNumRunnableApps() == allocConf.getQueueMaxApps(parent.getName()) - 1) {
        highestQueueWithAppsNowRunnable = parent;
      }
      parent = parent.getParent();
    }

    List<List<FSAppAttempt>> appsNowMaybeRunnable = new ArrayList<List<FSAppAttempt>>();

    // Compile lists of apps which may now be runnable
    // We gather lists instead of building a set of all non-runnable apps so
    // that this whole operation can be O(number of queues) instead of
    // O(number of apps)
    if (highestQueueWithAppsNowRunnable != null) {
      gatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable);
    }
    String user = app.getUser();
    Integer userNumRunning = usersNumRunnableApps.get(user);
    if (userNumRunning == null) {
      userNumRunning = 0;
    }
    if (userNumRunning == allocConf.getUserMaxApps(user) - 1) {
      List<FSAppAttempt> userWaitingApps = usersNonRunnableApps.get(user);
      if (userWaitingApps != null) {
        appsNowMaybeRunnable.add(userWaitingApps);
      }
    }

    updateAppsRunnability(appsNowMaybeRunnable, appsNowMaybeRunnable.size());
  }
  /**
   * Updates the relevant tracking variables after a runnable app with the given queue and user has
   * been removed.
   */
  public void untrackRunnableApp(FSAppAttempt app) {
    // Update usersRunnableApps
    String user = app.getUser();
    int newUserNumRunning = usersNumRunnableApps.get(user) - 1;
    if (newUserNumRunning == 0) {
      usersNumRunnableApps.remove(user);
    } else {
      usersNumRunnableApps.put(user, newUserNumRunning);
    }

    // Update runnable app bookkeeping for queues
    FSLeafQueue queue = app.getQueue();
    FSParentQueue parent = queue.getParent();
    while (parent != null) {
      parent.decrementRunnableApps();
      parent = parent.getParent();
    }
  }
 private FSAppAttempt addApp(FSLeafQueue queue, String user) {
   ApplicationId appId = ApplicationId.newInstance(0l, appNum++);
   ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 0);
   boolean runnable = maxAppsEnforcer.canAppBeRunnable(queue, user);
   FSAppAttempt app = new FSAppAttempt(scheduler, attId, user, queue, null, rmContext);
   queue.addApp(app, runnable);
   if (runnable) {
     maxAppsEnforcer.trackRunnableApp(app);
   } else {
     maxAppsEnforcer.trackNonRunnableApp(app);
   }
   return app;
 }
 @Test
 public void testRemoveEnablingOrderedByStartTime() {
   FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
   FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
   queueMaxApps.put("root.queue1", 2);
   FSAppAttempt app1 = addApp(leaf1, "user");
   addApp(leaf2, "user");
   addApp(leaf2, "user");
   clock.tick(20);
   addApp(leaf1, "user");
   assertEquals(1, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getRunnableAppSchedulables().size());
   assertEquals(1, leaf1.getNonRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
   removeApp(app1);
   assertEquals(0, leaf1.getRunnableAppSchedulables().size());
   assertEquals(2, leaf2.getRunnableAppSchedulables().size());
   assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
 }
 @Test
 public void testRemoveEnablesOneByQueueOneByUser() {
   FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.leaf1", true);
   FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.leaf2", true);
   queueMaxApps.put("root.queue1.leaf1", 2);
   userMaxApps.put("user1", 1);
   FSAppAttempt app1 = addApp(leaf1, "user1");
   addApp(leaf1, "user2");
   addApp(leaf1, "user3");
   addApp(leaf2, "user1");
   assertEquals(2, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf1.getNonRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
   removeApp(app1);
   assertEquals(2, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getRunnableAppSchedulables().size());
   assertEquals(0, leaf1.getNonRunnableAppSchedulables().size());
   assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
 }
 @Test
 public void testMultipleAppsWaitingOnCousinQueue() {
   FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
   FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
   queueMaxApps.put("root.queue1", 2);
   FSAppAttempt app1 = addApp(leaf1, "user");
   addApp(leaf2, "user");
   addApp(leaf2, "user");
   addApp(leaf2, "user");
   assertEquals(1, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getRunnableAppSchedulables().size());
   assertEquals(2, leaf2.getNonRunnableAppSchedulables().size());
   removeApp(app1);
   assertEquals(0, leaf1.getRunnableAppSchedulables().size());
   assertEquals(2, leaf2.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
 }
 @Test
 public void testRemoveDoesNotEnableAnyApp() {
   FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1", true);
   FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue2", true);
   queueMaxApps.put("root", 2);
   queueMaxApps.put("root.queue1", 1);
   queueMaxApps.put("root.queue2", 1);
   FSAppAttempt app1 = addApp(leaf1, "user");
   addApp(leaf2, "user");
   addApp(leaf2, "user");
   assertEquals(1, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
   removeApp(app1);
   assertEquals(0, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
 }
Example #10
0
 /** Get metrics reference from containing queue. */
 public QueueMetrics getMetrics() {
   return queue.getMetrics();
 }