/**
* print scheduling for debug purposes
*
* @param cluster
* @param topologies
*/
public static String printScheduling(Cluster cluster, Topologies topologies) {
StringBuilder str = new StringBuilder();
Map<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>> schedulingMap =
new HashMap<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>>();
for (TopologyDetails topo : topologies.getTopologies()) {
if (cluster.getAssignmentById(topo.getId()) != null) {
for (Map.Entry<ExecutorDetails, WorkerSlot> entry :
cluster.getAssignmentById(topo.getId()).getExecutorToSlot().entrySet()) {
WorkerSlot slot = entry.getValue();
String nodeId = slot.getNodeId();
ExecutorDetails exec = entry.getKey();
if (!schedulingMap.containsKey(nodeId)) {
schedulingMap.put(
nodeId, new HashMap<String, Map<WorkerSlot, Collection<ExecutorDetails>>>());
}
if (schedulingMap.get(nodeId).containsKey(topo.getId()) == false) {
schedulingMap
.get(nodeId)
.put(topo.getId(), new HashMap<WorkerSlot, Collection<ExecutorDetails>>());
}
if (schedulingMap.get(nodeId).get(topo.getId()).containsKey(slot) == false) {
schedulingMap
.get(nodeId)
.get(topo.getId())
.put(slot, new LinkedList<ExecutorDetails>());
}
schedulingMap.get(nodeId).get(topo.getId()).get(slot).add(exec);
}
}
}
for (Map.Entry<String, Map<String, Map<WorkerSlot, Collection<ExecutorDetails>>>> entry :
schedulingMap.entrySet()) {
if (cluster.getSupervisorById(entry.getKey()) != null) {
str.append(
"/** Node: "
+ cluster.getSupervisorById(entry.getKey()).getHost()
+ "-"
+ entry.getKey()
+ " **/\n");
} else {
str.append("/** Node: Unknown may be dead -" + entry.getKey() + " **/\n");
}
for (Map.Entry<String, Map<WorkerSlot, Collection<ExecutorDetails>>> topo_sched :
schedulingMap.get(entry.getKey()).entrySet()) {
str.append("\t-->Topology: " + topo_sched.getKey() + "\n");
for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> ws :
topo_sched.getValue().entrySet()) {
str.append("\t\t->Slot [" + ws.getKey().getPort() + "] -> " + ws.getValue() + "\n");
}
}
}
return str.toString();
}
public void linkBasedScheduling(
Topologies topos, Cluster cluster, GlobalState globalState, GlobalResources globalResources) {
for (TopologyDetails td : topos.getTopologies()) {
String topId = td.getId();
Map<Node, Collection<ExecutorDetails>> taskToNodesMap;
if (cluster.needsScheduling(td) && cluster.getUnassignedExecutors(td).size() > 0) {
LOG.info("/********Scheduling topology {} ************/", topId);
int totalTasks = td.getExecutors().size();
int executorsNotRunning = cluster.getUnassignedExecutors(td).size();
LOG.info(
"Total number of executors: {} " + "Total number of Unassigned Executors: {}",
totalTasks,
executorsNotRunning);
LOG.info("executors that need scheduling: {}", cluster.getUnassignedExecutors(td));
LinkBasedStrategy rs =
new LinkBasedStrategy(globalState, globalResources, null, td, cluster, topos);
taskToNodesMap = rs.schedule(td, cluster.getUnassignedExecutors(td));
if (taskToNodesMap != null) {
try {
for (Map.Entry<Node, Collection<ExecutorDetails>> entry : taskToNodesMap.entrySet()) {
entry.getKey().assign(td.getId(), entry.getValue(), cluster);
LOG.info(
"ASSIGNMENT TOPOLOGY: {} TASKS: {} To Node: "
+ entry.getKey().getId()
+ " Slots left: "
+ entry.getKey().totalSlotsFree(),
td.getId(),
entry.getValue());
}
LOG.info(
"Toplogy: {} assigned to {} nodes", td.getId(), taskToNodesMap.keySet().size());
HelperFuncs.setTopoStatus(td.getId(), "Fully Scheduled");
} catch (IllegalStateException ex) {
LOG.error(ex.toString());
LOG.error("Unsuccessfull in scheduling topology {}", td.getId());
HelperFuncs.setTopoStatus(td.getId(), "Unsuccessfull in scheduling topology");
}
} else {
LOG.error("Unsuccessfull in scheduling topology {}", td.getId());
HelperFuncs.setTopoStatus(td.getId(), "Unsuccessfull in scheduling topology");
}
} else {
HelperFuncs.setTopoStatus(td.getId(), "Fully Scheduled");
}
}
}