/**
* Breadth first traversal of the topology DAG
*
* @param topologies
* @param td
* @param spouts
* @return A partial ordering of components
*/
private Queue<Component> bfs(Topologies topologies, TopologyDetails td, List<Component> spouts) {
// Since queue is a interface
Queue<Component> ordered__Component_list = new LinkedList<Component>();
HashMap<String, Component> visited = new HashMap<String, Component>();
/* start from each spout that is not visited, each does a breadth-first traverse */
for (Component spout : spouts) {
if (!visited.containsKey(spout.id)) {
Queue<Component> queue = new LinkedList<Component>();
queue.offer(spout);
while (!queue.isEmpty()) {
Component comp = queue.poll();
visited.put(comp.id, comp);
ordered__Component_list.add(comp);
List<String> neighbors = new ArrayList<String>();
neighbors.addAll(comp.children);
neighbors.addAll(comp.parents);
for (String nbID : neighbors) {
if (!visited.containsKey(nbID)) {
Component child = topologies.getAllComponents().get(td.getId()).get(nbID);
queue.offer(child);
}
}
}
}
}
return ordered__Component_list;
}
private List<Component> getSpouts(Topologies topologies, TopologyDetails td) {
List<Component> spouts = new ArrayList<Component>();
for (Component c : topologies.getAllComponents().get(td.getId()).values()) {
if (c.type == Component.ComponentType.SPOUT) {
spouts.add(c);
}
}
return spouts;
}
/**
* print scheduling for debug purposes
*
* @param cluster
* @param topologies
*/
public static String printScheduling(Cluster cluster, Topologies topologies) {
StringBuilder str = new StringBuilder();
Map<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>> schedulingMap =
new HashMap<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>>();
for (TopologyDetails topo : topologies.getTopologies()) {
if (cluster.getAssignmentById(topo.getId()) != null) {
for (Map.Entry<ExecutorDetails, WorkerSlot> entry :
cluster.getAssignmentById(topo.getId()).getExecutorToSlot().entrySet()) {
WorkerSlot slot = entry.getValue();
String nodeId = slot.getNodeId();
ExecutorDetails exec = entry.getKey();
if (!schedulingMap.containsKey(nodeId)) {
schedulingMap.put(
nodeId, new HashMap<String, Map<WorkerSlot, Collection<ExecutorDetails>>>());
}
if (schedulingMap.get(nodeId).containsKey(topo.getId()) == false) {
schedulingMap
.get(nodeId)
.put(topo.getId(), new HashMap<WorkerSlot, Collection<ExecutorDetails>>());
}
if (schedulingMap.get(nodeId).get(topo.getId()).containsKey(slot) == false) {
schedulingMap
.get(nodeId)
.get(topo.getId())
.put(slot, new LinkedList<ExecutorDetails>());
}
schedulingMap.get(nodeId).get(topo.getId()).get(slot).add(exec);
}
}
}
for (Map.Entry<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>> entry :
schedulingMap.entrySet()) {
if (cluster.getSupervisorById(entry.getKey()) != null) {
str.append(
"/** Node: "
+ cluster.getSupervisorById(entry.getKey()).getHost()
+ "-"
+ entry.getKey()
+ " **/\n");
} else {
str.append("/** Node: Unknown may be dead -" + entry.getKey() + " **/\n");
}
for (Map.Entry<String, Map<WorkerSlot, Collection<ExecutorDetails>>> topo_sched :
schedulingMap.get(entry.getKey()).entrySet()) {
str.append("\t-->Topology: " + topo_sched.getKey() + "\n");
for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> ws :
topo_sched.getValue().entrySet()) {
str.append("\t\t->Slot [" + ws.getKey().getPort() + "] -> " + ws.getValue() + "\n");
}
}
}
return str.toString();
}
public void linkBasedScheduling(
Topologies topos, Cluster cluster, GlobalState globalState, GlobalResources globalResources) {
for (TopologyDetails td : topos.getTopologies()) {
String topId = td.getId();
Map<Node, Collection<ExecutorDetails>> taskToNodesMap;
if (cluster.needsScheduling(td) && cluster.getUnassignedExecutors(td).size() > 0) {
LOG.info("/********Scheduling topology {} ************/", topId);
int totalTasks = td.getExecutors().size();
int executorsNotRunning = cluster.getUnassignedExecutors(td).size();
LOG.info(
"Total number of executors: {} " + "Total number of Unassigned Executors: {}",
totalTasks,
executorsNotRunning);
LOG.info("executors that need scheduling: {}", cluster.getUnassignedExecutors(td));
LinkBasedStrategy rs =
new LinkBasedStrategy(globalState, globalResources, null, td, cluster, topos);
taskToNodesMap = rs.schedule(td, cluster.getUnassignedExecutors(td));
if (taskToNodesMap != null) {
try {
for (Map.Entry<Node, Collection<ExecutorDetails>> entry : taskToNodesMap.entrySet()) {
entry.getKey().assign(td.getId(), entry.getValue(), cluster);
LOG.info(
"ASSIGNMENT TOPOLOGY: {} TASKS: {} To Node: "
+ entry.getKey().getId()
+ " Slots left: "
+ entry.getKey().totalSlotsFree(),
td.getId(),
entry.getValue());
}
LOG.info(
"Toplogy: {} assigned to {} nodes", td.getId(), taskToNodesMap.keySet().size());
HelperFuncs.setTopoStatus(td.getId(), "Fully Scheduled");
} catch (IllegalStateException ex) {
LOG.error(ex.toString());
LOG.error("Unsuccessfull in scheduling topology {}", td.getId());
HelperFuncs.setTopoStatus(td.getId(), "Unsuccessfull in scheduling topology");
}
} else {
LOG.error("Unsuccessfull in scheduling topology {}", td.getId());
HelperFuncs.setTopoStatus(td.getId(), "Unsuccessfull in scheduling topology");
}
} else {
HelperFuncs.setTopoStatus(td.getId(), "Fully Scheduled");
}
}
}
