public Map<WorkerSlot, Collection<ExecutorDetails>> schedule(TopologyDetails td) {
if (_availNodes.size() <= 0) {
LOG.warn("No available nodes to schedule tasks on!");
return null;
}
Collection<ExecutorDetails> unassignedExecutors = _cluster.getUnassignedExecutors(td);
Map<WorkerSlot, Collection<ExecutorDetails>> schedulerAssignmentMap =
new HashMap<WorkerSlot, Collection<ExecutorDetails>>();
LOG.debug("ExecutorsNeedScheduling: {}", unassignedExecutors);
Collection<ExecutorDetails> scheduledTasks = new ArrayList<ExecutorDetails>();
List<Component> spouts = this.getSpouts(_topologies, td);
if (spouts.size() == 0) {
LOG.error("Cannot find a Spout!");
return null;
}
Queue<Component> ordered__Component_list = bfs(_topologies, td, spouts);
Map<Integer, List<ExecutorDetails>> priorityToExecutorMap =
getPriorityToExecutorDetailsListMap(ordered__Component_list, unassignedExecutors);
Collection<ExecutorDetails> executorsNotScheduled =
new HashSet<ExecutorDetails>(unassignedExecutors);
Integer longestPriorityListSize = this.getLongestPriorityListSize(priorityToExecutorMap);
// Pick the first executor with priority one, then the 1st exec with priority 2, so on an so
// forth.
// Once we reach the last priority, we go back to priority 1 and schedule the second task with
// priority 1.
for (int i = 0; i < longestPriorityListSize; i++) {
for (Entry<Integer, List<ExecutorDetails>> entry : priorityToExecutorMap.entrySet()) {
Iterator<ExecutorDetails> it = entry.getValue().iterator();
if (it.hasNext()) {
ExecutorDetails exec = it.next();
LOG.debug(
"\n\nAttempting to schedule: {} of component {}[avail {}] with rank {}",
new Object[] {
exec,
td.getExecutorToComponent().get(exec),
td.getTaskResourceReqList(exec),
entry.getKey()
});
WorkerSlot targetSlot = this.findWorkerForExec(exec, td, schedulerAssignmentMap);
if (targetSlot != null) {
RAS_Node targetNode = this.idToNode(targetSlot.getNodeId());
if (!schedulerAssignmentMap.containsKey(targetSlot)) {
schedulerAssignmentMap.put(targetSlot, new LinkedList<ExecutorDetails>());
}
schedulerAssignmentMap.get(targetSlot).add(exec);
targetNode.consumeResourcesforTask(exec, td);
scheduledTasks.add(exec);
LOG.debug(
"TASK {} assigned to Node: {} avail [mem: {} cpu: {}] total [mem: {} cpu: {}] on slot: {}",
exec,
targetNode,
targetNode.getAvailableMemoryResources(),
targetNode.getAvailableCpuResources(),
targetNode.getTotalMemoryResources(),
targetNode.getTotalCpuResources(),
targetSlot);
} else {
LOG.error("Not Enough Resources to schedule Task {}", exec);
}
it.remove();
}
}
}
executorsNotScheduled.removeAll(scheduledTasks);
LOG.debug("/* Scheduling left over task (most likely sys tasks) */");
// schedule left over system tasks
for (ExecutorDetails exec : executorsNotScheduled) {
WorkerSlot targetSlot = this.findWorkerForExec(exec, td, schedulerAssignmentMap);
if (targetSlot != null) {
RAS_Node targetNode = this.idToNode(targetSlot.getNodeId());
if (schedulerAssignmentMap.containsKey(targetSlot) == false) {
schedulerAssignmentMap.put(targetSlot, new LinkedList<ExecutorDetails>());
}
schedulerAssignmentMap.get(targetSlot).add(exec);
targetNode.consumeResourcesforTask(exec, td);
scheduledTasks.add(exec);
LOG.debug(
"TASK {} assigned to Node: {} avail [mem: {} cpu: {}] total [mem: {} cpu: {}] on slot: {}",
exec,
targetNode,
targetNode.getAvailableMemoryResources(),
targetNode.getAvailableCpuResources(),
targetNode.getTotalMemoryResources(),
targetNode.getTotalCpuResources(),
targetSlot);
} else {
LOG.error("Not Enough Resources to schedule Task {}", exec);
}
}
executorsNotScheduled.removeAll(scheduledTasks);
if (executorsNotScheduled.size() > 0) {
LOG.error("Not all executors successfully scheduled: {}", executorsNotScheduled);
schedulerAssignmentMap = null;
} else {
LOG.debug("All resources successfully scheduled!");
}
if (schedulerAssignmentMap == null) {
LOG.error("Topology {} not successfully scheduled!", td.getId());
}
return schedulerAssignmentMap;
}
