public Map<WorkerSlot, Collection<ExecutorDetails>> schedule(TopologyDetails td) { if (_availNodes.size() <= 0) { LOG.warn("No available nodes to schedule tasks on!"); return null; } Collection<ExecutorDetails> unassignedExecutors = _cluster.getUnassignedExecutors(td); Map<WorkerSlot, Collection<ExecutorDetails>> schedulerAssignmentMap = new HashMap<WorkerSlot, Collection<ExecutorDetails>>(); LOG.debug("ExecutorsNeedScheduling: {}", unassignedExecutors); Collection<ExecutorDetails> scheduledTasks = new ArrayList<ExecutorDetails>(); List<Component> spouts = this.getSpouts(_topologies, td); if (spouts.size() == 0) { LOG.error("Cannot find a Spout!"); return null; } Queue<Component> ordered__Component_list = bfs(_topologies, td, spouts); Map<Integer, List<ExecutorDetails>> priorityToExecutorMap = getPriorityToExecutorDetailsListMap(ordered__Component_list, unassignedExecutors); Collection<ExecutorDetails> executorsNotScheduled = new HashSet<ExecutorDetails>(unassignedExecutors); Integer longestPriorityListSize = this.getLongestPriorityListSize(priorityToExecutorMap); // Pick the first executor with priority one, then the 1st exec with priority 2, so on an so // forth. // Once we reach the last priority, we go back to priority 1 and schedule the second task with // priority 1. for (int i = 0; i < longestPriorityListSize; i++) { for (Entry<Integer, List<ExecutorDetails>> entry : priorityToExecutorMap.entrySet()) { Iterator<ExecutorDetails> it = entry.getValue().iterator(); if (it.hasNext()) { ExecutorDetails exec = it.next(); LOG.debug( "\n\nAttempting to schedule: {} of component {}[avail {}] with rank {}", new Object[] { exec, td.getExecutorToComponent().get(exec), td.getTaskResourceReqList(exec), entry.getKey() }); WorkerSlot targetSlot = this.findWorkerForExec(exec, td, schedulerAssignmentMap); if (targetSlot != null) { RAS_Node targetNode = this.idToNode(targetSlot.getNodeId()); if (!schedulerAssignmentMap.containsKey(targetSlot)) { schedulerAssignmentMap.put(targetSlot, new LinkedList<ExecutorDetails>()); } schedulerAssignmentMap.get(targetSlot).add(exec); targetNode.consumeResourcesforTask(exec, td); scheduledTasks.add(exec); LOG.debug( "TASK {} assigned to Node: {} avail [mem: {} cpu: {}] total [mem: {} cpu: {}] on slot: {}", exec, targetNode, targetNode.getAvailableMemoryResources(), targetNode.getAvailableCpuResources(), targetNode.getTotalMemoryResources(), targetNode.getTotalCpuResources(), targetSlot); } else { LOG.error("Not Enough Resources to schedule Task {}", exec); } it.remove(); } } } executorsNotScheduled.removeAll(scheduledTasks); LOG.debug("/* Scheduling left over task (most likely sys tasks) */"); // schedule left over system tasks for (ExecutorDetails exec : executorsNotScheduled) { WorkerSlot targetSlot = this.findWorkerForExec(exec, td, schedulerAssignmentMap); if (targetSlot != null) { RAS_Node targetNode = this.idToNode(targetSlot.getNodeId()); if (schedulerAssignmentMap.containsKey(targetSlot) == false) { schedulerAssignmentMap.put(targetSlot, new LinkedList<ExecutorDetails>()); } schedulerAssignmentMap.get(targetSlot).add(exec); targetNode.consumeResourcesforTask(exec, td); scheduledTasks.add(exec); LOG.debug( "TASK {} assigned to Node: {} avail [mem: {} cpu: {}] total [mem: {} cpu: {}] on slot: {}", exec, targetNode, targetNode.getAvailableMemoryResources(), targetNode.getAvailableCpuResources(), targetNode.getTotalMemoryResources(), targetNode.getTotalCpuResources(), targetSlot); } else { LOG.error("Not Enough Resources to schedule Task {}", exec); } } executorsNotScheduled.removeAll(scheduledTasks); if (executorsNotScheduled.size() > 0) { LOG.error("Not all executors successfully scheduled: {}", executorsNotScheduled); schedulerAssignmentMap = null; } else { LOG.debug("All resources successfully scheduled!"); } if (schedulerAssignmentMap == null) { LOG.error("Topology {} not successfully scheduled!", td.getId()); } return schedulerAssignmentMap; }