private WorkerSlot getBestWorker(
ExecutorDetails exec,
TopologyDetails td,
String clusterId,
Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) {
double taskMem = td.getTotalMemReqTask(exec);
double taskCPU = td.getTotalCpuReqTask(exec);
List<RAS_Node> nodes;
if (clusterId != null) {
nodes = this.getAvailableNodesFromCluster(clusterId);
} else {
nodes = this.getAvailableNodes();
}
// First sort nodes by distance
TreeMap<Double, RAS_Node> nodeRankMap = new TreeMap<Double, RAS_Node>();
for (RAS_Node n : nodes) {
if (n.getFreeSlots().size() > 0) {
if (n.getAvailableMemoryResources() >= taskMem && n.getAvailableCpuResources() >= taskCPU) {
double a =
Math.pow(
((taskCPU - n.getAvailableCpuResources()) / (n.getAvailableCpuResources() + 1))
* this.CPU_WEIGHT,
2);
double b =
Math.pow(
((taskMem - n.getAvailableMemoryResources())
/ (n.getAvailableMemoryResources() + 1))
* this.MEM_WEIGHT,
2);
double c = 0.0;
if (this.refNode != null) {
c = Math.pow(this.distToNode(this.refNode, n) * this.NETWORK_WEIGHT, 2);
}
double distance = Math.sqrt(a + b + c);
nodeRankMap.put(distance, n);
}
}
}
// Then, pick worker from closest node that satisfy constraints
for (Map.Entry<Double, RAS_Node> entry : nodeRankMap.entrySet()) {
RAS_Node n = entry.getValue();
for (WorkerSlot ws : n.getFreeSlots()) {
if (checkWorkerConstraints(exec, ws, td, scheduleAssignmentMap)) {
return ws;
}
}
}
return null;
}
/**
* Get the remaining amount memory that can be assigned to a worker given the set worker max heap
* size
*
* @param ws
* @param td
* @param scheduleAssignmentMap
* @return The remaining amount of memory
*/
private Double getWorkerScheduledMemoryAvailable(
WorkerSlot ws,
TopologyDetails td,
Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) {
Double memScheduleUsed = this.getWorkerScheduledMemoryUse(ws, td, scheduleAssignmentMap);
return td.getTopologyWorkerMaxHeapSize() - memScheduleUsed;
}
/**
* Breadth first traversal of the topology DAG
*
* @param topologies
* @param td
* @param spouts
* @return A partial ordering of components
*/
private Queue<Component> bfs(Topologies topologies, TopologyDetails td, List<Component> spouts) {
// Since queue is a interface
Queue<Component> ordered__Component_list = new LinkedList<Component>();
HashMap<String, Component> visited = new HashMap<String, Component>();
/* start from each spout that is not visited, each does a breadth-first traverse */
for (Component spout : spouts) {
if (!visited.containsKey(spout.id)) {
Queue<Component> queue = new LinkedList<Component>();
queue.offer(spout);
while (!queue.isEmpty()) {
Component comp = queue.poll();
visited.put(comp.id, comp);
ordered__Component_list.add(comp);
List<String> neighbors = new ArrayList<String>();
neighbors.addAll(comp.children);
neighbors.addAll(comp.parents);
for (String nbID : neighbors) {
if (!visited.containsKey(nbID)) {
Component child = topologies.getAllComponents().get(td.getId()).get(nbID);
queue.offer(child);
}
}
}
}
}
return ordered__Component_list;
}
private List<Component> getSpouts(Topologies topologies, TopologyDetails td) {
List<Component> spouts = new ArrayList<Component>();
for (Component c : topologies.getAllComponents().get(td.getId()).values()) {
if (c.type == Component.ComponentType.SPOUT) {
spouts.add(c);
}
}
return spouts;
}
/**
* print scheduling for debug purposes
*
* @param cluster
* @param topologies
*/
public static String printScheduling(Cluster cluster, Topologies topologies) {
StringBuilder str = new StringBuilder();
Map<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>> schedulingMap =
new HashMap<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>>();
for (TopologyDetails topo : topologies.getTopologies()) {
if (cluster.getAssignmentById(topo.getId()) != null) {
for (Map.Entry<ExecutorDetails, WorkerSlot> entry :
cluster.getAssignmentById(topo.getId()).getExecutorToSlot().entrySet()) {
WorkerSlot slot = entry.getValue();
String nodeId = slot.getNodeId();
ExecutorDetails exec = entry.getKey();
if (!schedulingMap.containsKey(nodeId)) {
schedulingMap.put(
nodeId, new HashMap<String, Map<WorkerSlot, Collection<ExecutorDetails>>>());
}
if (schedulingMap.get(nodeId).containsKey(topo.getId()) == false) {
schedulingMap
.get(nodeId)
.put(topo.getId(), new HashMap<WorkerSlot, Collection<ExecutorDetails>>());
}
if (schedulingMap.get(nodeId).get(topo.getId()).containsKey(slot) == false) {
schedulingMap
.get(nodeId)
.get(topo.getId())
.put(slot, new LinkedList<ExecutorDetails>());
}
schedulingMap.get(nodeId).get(topo.getId()).get(slot).add(exec);
}
}
}
for (Map.Entry<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>> entry :
schedulingMap.entrySet()) {
if (cluster.getSupervisorById(entry.getKey()) != null) {
str.append(
"/** Node: "
+ cluster.getSupervisorById(entry.getKey()).getHost()
+ "-"
+ entry.getKey()
+ " **/\n");
} else {
str.append("/** Node: Unknown may be dead -" + entry.getKey() + " **/\n");
}
for (Map.Entry<String, Map<WorkerSlot, Collection<ExecutorDetails>>> topo_sched :
schedulingMap.get(entry.getKey()).entrySet()) {
str.append("\t-->Topology: " + topo_sched.getKey() + "\n");
for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> ws :
topo_sched.getValue().entrySet()) {
str.append("\t\t->Slot [" + ws.getKey().getPort() + "] -> " + ws.getValue() + "\n");
}
}
}
return str.toString();
}
/**
* Checks whether we can schedule an Executor exec on the worker slot ws Only considers memory
* currenlty. May include CPU in the future
*
* @param exec
* @param ws
* @param td
* @param scheduleAssignmentMap
* @return a boolean: True denoting the exec can be scheduled on ws and false if it cannot
*/
private boolean checkWorkerConstraints(
ExecutorDetails exec,
WorkerSlot ws,
TopologyDetails td,
Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) {
boolean retVal = false;
if (this.getWorkerScheduledMemoryAvailable(ws, td, scheduleAssignmentMap)
>= td.getTotalMemReqTask(exec)) {
retVal = true;
}
return retVal;
}
/**
* Get the amount of memory already assigned to a worker
*
* @param ws
* @param td
* @param scheduleAssignmentMap
* @return the amount of memory
*/
private Double getWorkerScheduledMemoryUse(
WorkerSlot ws,
TopologyDetails td,
Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) {
Double totalMem = 0.0;
Collection<ExecutorDetails> execs = scheduleAssignmentMap.get(ws);
if (execs != null) {
for (ExecutorDetails exec : execs) {
totalMem += td.getTotalMemReqTask(exec);
}
}
return totalMem;
}
public Map<WorkerSlot, Collection<ExecutorDetails>> schedule(TopologyDetails td) {
if (_availNodes.size() <= 0) {
LOG.warn("No available nodes to schedule tasks on!");
return null;
}
Collection<ExecutorDetails> unassignedExecutors = _cluster.getUnassignedExecutors(td);
Map<WorkerSlot, Collection<ExecutorDetails>> schedulerAssignmentMap =
new HashMap<WorkerSlot, Collection<ExecutorDetails>>();
LOG.debug("ExecutorsNeedScheduling: {}", unassignedExecutors);
Collection<ExecutorDetails> scheduledTasks = new ArrayList<ExecutorDetails>();
List<Component> spouts = this.getSpouts(_topologies, td);
if (spouts.size() == 0) {
LOG.error("Cannot find a Spout!");
return null;
}
Queue<Component> ordered__Component_list = bfs(_topologies, td, spouts);
Map<Integer, List<ExecutorDetails>> priorityToExecutorMap =
getPriorityToExecutorDetailsListMap(ordered__Component_list, unassignedExecutors);
Collection<ExecutorDetails> executorsNotScheduled =
new HashSet<ExecutorDetails>(unassignedExecutors);
Integer longestPriorityListSize = this.getLongestPriorityListSize(priorityToExecutorMap);
// Pick the first executor with priority one, then the 1st exec with priority 2, so on an so
// forth.
// Once we reach the last priority, we go back to priority 1 and schedule the second task with
// priority 1.
for (int i = 0; i < longestPriorityListSize; i++) {
for (Entry<Integer, List<ExecutorDetails>> entry : priorityToExecutorMap.entrySet()) {
Iterator<ExecutorDetails> it = entry.getValue().iterator();
if (it.hasNext()) {
ExecutorDetails exec = it.next();
LOG.debug(
"\n\nAttempting to schedule: {} of component {}[avail {}] with rank {}",
new Object[] {
exec,
td.getExecutorToComponent().get(exec),
td.getTaskResourceReqList(exec),
entry.getKey()
});
WorkerSlot targetSlot = this.findWorkerForExec(exec, td, schedulerAssignmentMap);
if (targetSlot != null) {
RAS_Node targetNode = this.idToNode(targetSlot.getNodeId());
if (!schedulerAssignmentMap.containsKey(targetSlot)) {
schedulerAssignmentMap.put(targetSlot, new LinkedList<ExecutorDetails>());
}
schedulerAssignmentMap.get(targetSlot).add(exec);
targetNode.consumeResourcesforTask(exec, td);
scheduledTasks.add(exec);
LOG.debug(
"TASK {} assigned to Node: {} avail [mem: {} cpu: {}] total [mem: {} cpu: {}] on slot: {}",
exec,
targetNode,
targetNode.getAvailableMemoryResources(),
targetNode.getAvailableCpuResources(),
targetNode.getTotalMemoryResources(),
targetNode.getTotalCpuResources(),
targetSlot);
} else {
LOG.error("Not Enough Resources to schedule Task {}", exec);
}
it.remove();
}
}
}
executorsNotScheduled.removeAll(scheduledTasks);
LOG.debug("/* Scheduling left over task (most likely sys tasks) */");
// schedule left over system tasks
for (ExecutorDetails exec : executorsNotScheduled) {
WorkerSlot targetSlot = this.findWorkerForExec(exec, td, schedulerAssignmentMap);
if (targetSlot != null) {
RAS_Node targetNode = this.idToNode(targetSlot.getNodeId());
if (schedulerAssignmentMap.containsKey(targetSlot) == false) {
schedulerAssignmentMap.put(targetSlot, new LinkedList<ExecutorDetails>());
}
schedulerAssignmentMap.get(targetSlot).add(exec);
targetNode.consumeResourcesforTask(exec, td);
scheduledTasks.add(exec);
LOG.debug(
"TASK {} assigned to Node: {} avail [mem: {} cpu: {}] total [mem: {} cpu: {}] on slot: {}",
exec,
targetNode,
targetNode.getAvailableMemoryResources(),
targetNode.getAvailableCpuResources(),
targetNode.getTotalMemoryResources(),
targetNode.getTotalCpuResources(),
targetSlot);
} else {
LOG.error("Not Enough Resources to schedule Task {}", exec);
}
}
executorsNotScheduled.removeAll(scheduledTasks);
if (executorsNotScheduled.size() > 0) {
LOG.error("Not all executors successfully scheduled: {}", executorsNotScheduled);
schedulerAssignmentMap = null;
} else {
LOG.debug("All resources successfully scheduled!");
}
if (schedulerAssignmentMap == null) {
LOG.error("Topology {} not successfully scheduled!", td.getId());
}
return schedulerAssignmentMap;
}
public void linkBasedScheduling(
Topologies topos, Cluster cluster, GlobalState globalState, GlobalResources globalResources) {
for (TopologyDetails td : topos.getTopologies()) {
String topId = td.getId();
Map<Node, Collection<ExecutorDetails>> taskToNodesMap;
if (cluster.needsScheduling(td) && cluster.getUnassignedExecutors(td).size() > 0) {
LOG.info("/********Scheduling topology {} ************/", topId);
int totalTasks = td.getExecutors().size();
int executorsNotRunning = cluster.getUnassignedExecutors(td).size();
LOG.info(
"Total number of executors: {} " + "Total number of Unassigned Executors: {}",
totalTasks,
executorsNotRunning);
LOG.info("executors that need scheduling: {}", cluster.getUnassignedExecutors(td));
LinkBasedStrategy rs =
new LinkBasedStrategy(globalState, globalResources, null, td, cluster, topos);
taskToNodesMap = rs.schedule(td, cluster.getUnassignedExecutors(td));
if (taskToNodesMap != null) {
try {
for (Map.Entry<Node, Collection<ExecutorDetails>> entry : taskToNodesMap.entrySet()) {
entry.getKey().assign(td.getId(), entry.getValue(), cluster);
LOG.info(
"ASSIGNMENT TOPOLOGY: {} TASKS: {} To Node: "
+ entry.getKey().getId()
+ " Slots left: "
+ entry.getKey().totalSlotsFree(),
td.getId(),
entry.getValue());
}
LOG.info(
"Toplogy: {} assigned to {} nodes", td.getId(), taskToNodesMap.keySet().size());
HelperFuncs.setTopoStatus(td.getId(), "Fully Scheduled");
} catch (IllegalStateException ex) {
LOG.error(ex.toString());
LOG.error("Unsuccessfull in scheduling topology {}", td.getId());
HelperFuncs.setTopoStatus(td.getId(), "Unsuccessfull in scheduling topology");
}
} else {
LOG.error("Unsuccessfull in scheduling topology {}", td.getId());
HelperFuncs.setTopoStatus(td.getId(), "Unsuccessfull in scheduling topology");
}
} else {
HelperFuncs.setTopoStatus(td.getId(), "Fully Scheduled");
}
}
}