private ComponentAssignType getComponentType( DefaultTopologyAssignContext defaultContext, Integer taskId) { StormTopology sysTopology = defaultContext.getSysTopology(); final Map<Integer, String> taskToComponent = defaultContext.getTaskToComponent(); String componentName = taskToComponent.get(taskId); if (componentName == null) { LOG.warn("No component name of " + taskId); return ComponentAssignType.NORMAL; } ComponentCommon common = ThriftTopologyUtils.getComponentCommon(sysTopology, componentName); String jsonConfString = common.get_json_conf(); if (jsonConfString == null) { return ComponentAssignType.NORMAL; } Map componentMap = new HashMap(); componentMap.putAll((Map) JStormUtils.from_json(jsonConfString)); if (JStormServerConfig.getUserDefineAssignmentFromJson(componentMap) != null) { // user define assignment return ComponentAssignType.USER_DEFINE; } else if (ConfigExtension.isUseOldAssignment(componentMap)) { // use old assignment return ComponentAssignType.USE_OLD; } else if (ConfigExtension.isUseOldAssignment(defaultContext.getStormConf())) { // use old assignment return ComponentAssignType.USE_OLD; } else { return ComponentAssignType.NORMAL; } }
private Map<String, Integer> computeComponentWeight( StormTopology rawTopology, DefaultTopologyAssignContext context) { Map<String, Integer> ret = new HashMap<String, Integer>(); Map<String, Object> components = ThriftTopologyUtils.getComponents(rawTopology); for (Entry<String, Object> entry : components.entrySet()) { String componentName = entry.getKey(); Object component = entry.getValue(); ComponentCommon common = null; if (component instanceof Bolt) { common = ((Bolt) component).get_common(); } if (component instanceof SpoutSpec) { common = ((SpoutSpec) component).get_common(); } if (component instanceof StateSpoutSpec) { common = ((StateSpoutSpec) component).get_common(); } String jsonConfString = common.get_json_conf(); if (jsonConfString == null) { ret.put(componentName, context.DEFAULT_WEIGHT); continue; } Map componentMap = new HashMap(); componentMap.putAll((Map) JStormUtils.from_json(jsonConfString)); int weight = computeWeight(componentMap, context); ret.put(componentName, weight); } return ret; }
@Override public Map<Integer, ResourceAssignment> assignTasks(TopologyAssignContext context) throws FailedAssignTopologyException { int assignType = context.getAssignType(); if (TopologyAssignContext.isAssignTypeValid(assignType) == false) { throw new FailedAssignTopologyException("Invalide Assign Type " + assignType); } DefaultTopologyAssignContext defaultContext = new DefaultTopologyAssignContext(context); if (assignType == TopologyAssignContext.ASSIGN_TYPE_REBALANCE) { freeUsed(defaultContext); } LOG.info("Dead tasks:" + defaultContext.getDeadTaskIds()); LOG.info("Unstopped tasks:" + defaultContext.getUnstoppedTaskIds()); Set<Integer> needAssignTasks = getNeedAssignTasks(defaultContext); Map<Integer, ResourceAssignment> keepAssigns = getKeepAssign(defaultContext, needAssignTasks); // please use tree map to make task sequence Map<Integer, ResourceAssignment> ret = new TreeMap<Integer, ResourceAssignment>(); ret.putAll(keepAssigns); ret.putAll(defaultContext.getUnstoppedAssignments()); Map<WorkerSlot, List<Integer>> keepAssignWorkers = Assignment.getWorkerTasks(keepAssigns); int allocWorkerNum = defaultContext.getTotalWorkerNum() - defaultContext.getUnstoppedWorkerNum() - keepAssignWorkers.size(); if (allocWorkerNum <= 0) { LOG.warn( "Don't need assign workers, all workers are fine " + defaultContext.toDetailString()); throw new FailedAssignTopologyException("Don't need assign worker, all workers are fine "); } Set<String> outputConfigComponents = new HashSet<String>(); Map<ComponentAssignType, Pair<Set<Integer>, IPreassignTask>> typeHandler = registerPreAssignHandler(defaultContext, needAssignTasks); Map<Integer, ResourceAssignment> newAssigns = new HashMap<Integer, ResourceAssignment>(); Set<String> usedSupervisorIds = new HashSet<String>(); List<Integer> lastFailed = new ArrayList<Integer>(); for (Entry<ComponentAssignType, Pair<Set<Integer>, IPreassignTask>> entry : typeHandler.entrySet()) { ComponentAssignType type = entry.getKey(); Set<Integer> tasks = entry.getValue().getFirst(); IPreassignTask handler = entry.getValue().getSecond(); tasks.addAll(lastFailed); lastFailed.clear(); List<Integer> sortedTasks = sortAssignTasks(defaultContext, tasks); StormTopology sysTopology = defaultContext.getSysTopology(); for (Integer task : sortedTasks) { Set<String> canUsedSupervisorIds = getCanUsedSupervisors(defaultContext, usedSupervisorIds, allocWorkerNum); String componentName = defaultContext.getTaskToComponent().get(task); ComponentCommon componentCommon = ThriftTopologyUtils.getComponentCommon(sysTopology, componentName); Map componentMap = (Map) JStormUtils.from_json(componentCommon.get_json_conf()); if (componentMap == null) { componentMap = Maps.newHashMap(); } if (outputConfigComponents.contains(componentName) == false) { LOG.info("Component map of " + componentName + "\n" + componentMap); outputConfigComponents.add(componentName); } ResourceAssignment preAssignment = handler.preAssign( task, defaultContext, componentMap, componentName, canUsedSupervisorIds, ret, newAssigns); if (preAssignment == null) { // pre assign fail lastFailed.add(task); } else { // sucess to do preAssign SupervisorInfo supervisorInfo = defaultContext.getCluster().get(preAssignment.getSupervisorId()); LOG.info("Task " + task + " had been assigned to " + supervisorInfo.getHostName()); newAssigns.put(task, preAssignment); ret.put(task, preAssignment); usedSupervisorIds.add(preAssignment.getSupervisorId()); } } } if (lastFailed.isEmpty() == false) { throw new FailedAssignTopologyException("Failed to assign tasks " + lastFailed); } // Here just hardcode IPostAssignTask postAssignHandler = new PostAssignTaskPort(); postAssignHandler.postAssign(defaultContext, newAssigns, allocWorkerNum); LOG.info("Keep Alive slots:" + keepAssigns); LOG.info("Unstopped slots:" + defaultContext.getUnstoppedAssignments()); LOG.info("New assign slots:" + newAssigns); return ret; }