/** Get unstopped slots from alive task list 获得所有supervisor已经dead,但是还没有dead的任务 */ public Set<Integer> getUnstoppedSlots( Set<Integer> aliveTasks, Map<String, SupervisorInfo> supInfos, Assignment existAssignment) { Set<Integer> ret = new HashSet<Integer>(); Set<ResourceWorkerSlot> oldWorkers = existAssignment.getWorkers(); Set<String> aliveSupervisors = supInfos.keySet(); for (ResourceWorkerSlot worker : oldWorkers) { for (Integer taskId : worker.getTasks()) { if (aliveTasks.contains(taskId) == false) { // task is dead continue; } String oldTaskSupervisorId = worker.getNodeId(); if (aliveSupervisors.contains(oldTaskSupervisorId) == false) { // supervisor is dead ret.add(taskId); continue; } } } return ret; }
/** * get all taskids which should be reassigned * * @param taskToWorkerSlot * @param newtaskToWorkerSlot * @return Set<Integer> taskid which should reassigned */ public static Set<Integer> getChangeTaskIds( Set<ResourceWorkerSlot> oldWorkers, Set<ResourceWorkerSlot> workers) { Set<Integer> rtn = new HashSet<Integer>(); for (ResourceWorkerSlot worker : workers) { if (!oldWorkers.contains(worker)) rtn.addAll(worker.getTasks()); } return rtn; }
private Set<ResourceWorkerSlot> getUnstoppedWorkers( Set<Integer> aliveTasks, Assignment existAssignment) { Set<ResourceWorkerSlot> ret = new HashSet<ResourceWorkerSlot>(); for (ResourceWorkerSlot worker : existAssignment.getWorkers()) { boolean alive = true; for (Integer task : worker.getTasks()) { if (!aliveTasks.contains(task)) { alive = false; break; } } if (alive) { ret.add(worker); } } return ret; }
/** * @param existingAssignment * @param taskWorkerSlot * @return * @throws Exception */ public static Map<Integer, Integer> getTaskStartTimes( TopologyAssignContext context, NimbusData nimbusData, String topologyId, Assignment existingAssignment, Set<ResourceWorkerSlot> workers) throws Exception { Map<Integer, Integer> startTimes = new TreeMap<Integer, Integer>(); if (context.getAssignType() == TopologyAssignContext.ASSIGN_TYPE_NEW) { int nowSecs = TimeUtils.current_time_secs(); for (ResourceWorkerSlot worker : workers) { for (Integer changedTaskId : worker.getTasks()) { startTimes.put(changedTaskId, nowSecs); } } return startTimes; } Set<ResourceWorkerSlot> oldWorkers = new HashSet<ResourceWorkerSlot>(); if (existingAssignment != null) { Map<Integer, Integer> taskStartTimeSecs = existingAssignment.getTaskStartTimeSecs(); if (taskStartTimeSecs != null) { startTimes.putAll(taskStartTimeSecs); } if (existingAssignment.getWorkers() != null) { oldWorkers = existingAssignment.getWorkers(); } } StormClusterState zkClusterState = nimbusData.getStormClusterState(); Set<Integer> changeTaskIds = getChangeTaskIds(oldWorkers, workers); int nowSecs = TimeUtils.current_time_secs(); for (Integer changedTaskId : changeTaskIds) { startTimes.put(changedTaskId, nowSecs); zkClusterState.remove_task_heartbeat(topologyId, changedTaskId); } LOG.info("Task assignment has been changed " + changeTaskIds); return startTimes; }
private static Set<ResourceWorkerSlot> mkLocalAssignment(TopologyAssignContext context) { Set<ResourceWorkerSlot> result = new HashSet<ResourceWorkerSlot>(); Map<String, SupervisorInfo> cluster = context.getCluster(); if (cluster.size() != 1) throw new RuntimeException(); SupervisorInfo localSupervisor = null; String supervisorId = null; for (Entry<String, SupervisorInfo> entry : cluster.entrySet()) { supervisorId = entry.getKey(); localSupervisor = entry.getValue(); } int port = localSupervisor.getWorkerPorts().iterator().next(); ResourceWorkerSlot worker = new ResourceWorkerSlot(supervisorId, port); worker.setTasks(new HashSet<Integer>(context.getAllTaskIds())); worker.setHostname(localSupervisor.getHostName()); result.add(worker); return result; }
/** * Get free resources 获得所有的空闲资源 * * @param supervisorInfos * @param stormClusterState * @throws Exception */ public static void getFreeSlots( Map<String, SupervisorInfo> supervisorInfos, StormClusterState stormClusterState) throws Exception { Map<String, Assignment> assignments = Cluster.get_all_assignment(stormClusterState, null); for (Entry<String, Assignment> entry : assignments.entrySet()) { String topologyId = entry.getKey(); Assignment assignment = entry.getValue(); Set<ResourceWorkerSlot> workers = assignment.getWorkers(); for (ResourceWorkerSlot worker : workers) { SupervisorInfo supervisorInfo = supervisorInfos.get(worker.getNodeId()); if (supervisorInfo == null) { // the supervisor is dead continue; } supervisorInfo.getWorkerPorts().remove(worker.getPort()); } } }
public static Map<String, String> getTopologyNodeHost( Map<String, SupervisorInfo> supervisorMap, Assignment existingAssignment, Set<ResourceWorkerSlot> workers) { // the following is that remove unused node from allNodeHost Set<String> usedNodes = new HashSet<String>(); for (ResourceWorkerSlot worker : workers) { usedNodes.add(worker.getNodeId()); } // map<supervisorId, hostname> Map<String, String> allNodeHost = new HashMap<String, String>(); if (existingAssignment != null) { allNodeHost.putAll(existingAssignment.getNodeHost()); } // get alive supervisorMap Map<supervisorId, hostname> Map<String, String> nodeHost = SupervisorInfo.getNodeHost(supervisorMap); if (nodeHost != null) { allNodeHost.putAll(nodeHost); } Map<String, String> ret = new HashMap<String, String>(); for (String supervisorId : usedNodes) { if (allNodeHost.containsKey(supervisorId)) { ret.put(supervisorId, allNodeHost.get(supervisorId)); } else { LOG.warn("Node " + supervisorId + " doesn't in the supervisor list"); } } return ret; }