private ComponentAssignType getComponentType(
DefaultTopologyAssignContext defaultContext, Integer taskId) {
StormTopology sysTopology = defaultContext.getSysTopology();
final Map<Integer, String> taskToComponent = defaultContext.getTaskToComponent();
String componentName = taskToComponent.get(taskId);
if (componentName == null) {
LOG.warn("No component name of " + taskId);
return ComponentAssignType.NORMAL;
}
ComponentCommon common = ThriftTopologyUtils.getComponentCommon(sysTopology, componentName);
String jsonConfString = common.get_json_conf();
if (jsonConfString == null) {
return ComponentAssignType.NORMAL;
}
Map componentMap = new HashMap();
componentMap.putAll((Map) JStormUtils.from_json(jsonConfString));
if (JStormServerConfig.getUserDefineAssignmentFromJson(componentMap) != null) {
// user define assignment
return ComponentAssignType.USER_DEFINE;
} else if (ConfigExtension.isUseOldAssignment(componentMap)) {
// use old assignment
return ComponentAssignType.USE_OLD;
} else if (ConfigExtension.isUseOldAssignment(defaultContext.getStormConf())) {
// use old assignment
return ComponentAssignType.USE_OLD;
} else {
return ComponentAssignType.NORMAL;
}
}
private Map<String, Integer> computeComponentWeight(
StormTopology rawTopology, DefaultTopologyAssignContext context) {
Map<String, Integer> ret = new HashMap<String, Integer>();
Map<String, Object> components = ThriftTopologyUtils.getComponents(rawTopology);
for (Entry<String, Object> entry : components.entrySet()) {
String componentName = entry.getKey();
Object component = entry.getValue();
ComponentCommon common = null;
if (component instanceof Bolt) {
common = ((Bolt) component).get_common();
}
if (component instanceof SpoutSpec) {
common = ((SpoutSpec) component).get_common();
}
if (component instanceof StateSpoutSpec) {
common = ((StateSpoutSpec) component).get_common();
}
String jsonConfString = common.get_json_conf();
if (jsonConfString == null) {
ret.put(componentName, context.DEFAULT_WEIGHT);
continue;
}
Map componentMap = new HashMap();
componentMap.putAll((Map) JStormUtils.from_json(jsonConfString));
int weight = computeWeight(componentMap, context);
ret.put(componentName, weight);
}
return ret;
}
@Override
public Map<Integer, ResourceAssignment> assignTasks(TopologyAssignContext context)
throws FailedAssignTopologyException {
int assignType = context.getAssignType();
if (TopologyAssignContext.isAssignTypeValid(assignType) == false) {
throw new FailedAssignTopologyException("Invalide Assign Type " + assignType);
}
DefaultTopologyAssignContext defaultContext = new DefaultTopologyAssignContext(context);
if (assignType == TopologyAssignContext.ASSIGN_TYPE_REBALANCE) {
freeUsed(defaultContext);
}
LOG.info("Dead tasks:" + defaultContext.getDeadTaskIds());
LOG.info("Unstopped tasks:" + defaultContext.getUnstoppedTaskIds());
Set<Integer> needAssignTasks = getNeedAssignTasks(defaultContext);
Map<Integer, ResourceAssignment> keepAssigns = getKeepAssign(defaultContext, needAssignTasks);
// please use tree map to make task sequence
Map<Integer, ResourceAssignment> ret = new TreeMap<Integer, ResourceAssignment>();
ret.putAll(keepAssigns);
ret.putAll(defaultContext.getUnstoppedAssignments());
Map<WorkerSlot, List<Integer>> keepAssignWorkers = Assignment.getWorkerTasks(keepAssigns);
int allocWorkerNum =
defaultContext.getTotalWorkerNum()
- defaultContext.getUnstoppedWorkerNum()
- keepAssignWorkers.size();
if (allocWorkerNum <= 0) {
LOG.warn(
"Don't need assign workers, all workers are fine " + defaultContext.toDetailString());
throw new FailedAssignTopologyException("Don't need assign worker, all workers are fine ");
}
Set<String> outputConfigComponents = new HashSet<String>();
Map<ComponentAssignType, Pair<Set<Integer>, IPreassignTask>> typeHandler =
registerPreAssignHandler(defaultContext, needAssignTasks);
Map<Integer, ResourceAssignment> newAssigns = new HashMap<Integer, ResourceAssignment>();
Set<String> usedSupervisorIds = new HashSet<String>();
List<Integer> lastFailed = new ArrayList<Integer>();
for (Entry<ComponentAssignType, Pair<Set<Integer>, IPreassignTask>> entry :
typeHandler.entrySet()) {
ComponentAssignType type = entry.getKey();
Set<Integer> tasks = entry.getValue().getFirst();
IPreassignTask handler = entry.getValue().getSecond();
tasks.addAll(lastFailed);
lastFailed.clear();
List<Integer> sortedTasks = sortAssignTasks(defaultContext, tasks);
StormTopology sysTopology = defaultContext.getSysTopology();
for (Integer task : sortedTasks) {
Set<String> canUsedSupervisorIds =
getCanUsedSupervisors(defaultContext, usedSupervisorIds, allocWorkerNum);
String componentName = defaultContext.getTaskToComponent().get(task);
ComponentCommon componentCommon =
ThriftTopologyUtils.getComponentCommon(sysTopology, componentName);
Map componentMap = (Map) JStormUtils.from_json(componentCommon.get_json_conf());
if (componentMap == null) {
componentMap = Maps.newHashMap();
}
if (outputConfigComponents.contains(componentName) == false) {
LOG.info("Component map of " + componentName + "\n" + componentMap);
outputConfigComponents.add(componentName);
}
ResourceAssignment preAssignment =
handler.preAssign(
task,
defaultContext,
componentMap,
componentName,
canUsedSupervisorIds,
ret,
newAssigns);
if (preAssignment == null) {
// pre assign fail
lastFailed.add(task);
} else {
// sucess to do preAssign
SupervisorInfo supervisorInfo =
defaultContext.getCluster().get(preAssignment.getSupervisorId());
LOG.info("Task " + task + " had been assigned to " + supervisorInfo.getHostName());
newAssigns.put(task, preAssignment);
ret.put(task, preAssignment);
usedSupervisorIds.add(preAssignment.getSupervisorId());
}
}
}
if (lastFailed.isEmpty() == false) {
throw new FailedAssignTopologyException("Failed to assign tasks " + lastFailed);
}
// Here just hardcode
IPostAssignTask postAssignHandler = new PostAssignTaskPort();
postAssignHandler.postAssign(defaultContext, newAssigns, allocWorkerNum);
LOG.info("Keep Alive slots:" + keepAssigns);
LOG.info("Unstopped slots:" + defaultContext.getUnstoppedAssignments());
LOG.info("New assign slots:" + newAssigns);
return ret;
}
