private ComponentAssignType getComponentType(
DefaultTopologyAssignContext defaultContext, Integer taskId) {
StormTopology sysTopology = defaultContext.getSysTopology();
final Map<Integer, String> taskToComponent = defaultContext.getTaskToComponent();
String componentName = taskToComponent.get(taskId);
if (componentName == null) {
LOG.warn("No component name of " + taskId);
return ComponentAssignType.NORMAL;
}
ComponentCommon common = ThriftTopologyUtils.getComponentCommon(sysTopology, componentName);
String jsonConfString = common.get_json_conf();
if (jsonConfString == null) {
return ComponentAssignType.NORMAL;
}
Map componentMap = new HashMap();
componentMap.putAll((Map) JStormUtils.from_json(jsonConfString));
if (JStormServerConfig.getUserDefineAssignmentFromJson(componentMap) != null) {
// user define assignment
return ComponentAssignType.USER_DEFINE;
} else if (ConfigExtension.isUseOldAssignment(componentMap)) {
// use old assignment
return ComponentAssignType.USE_OLD;
} else if (ConfigExtension.isUseOldAssignment(defaultContext.getStormConf())) {
// use old assignment
return ComponentAssignType.USE_OLD;
} else {
return ComponentAssignType.NORMAL;
}
}
private Map<String, Integer> computeComponentWeight(
StormTopology rawTopology, DefaultTopologyAssignContext context) {
Map<String, Integer> ret = new HashMap<String, Integer>();
Map<String, Object> components = ThriftTopologyUtils.getComponents(rawTopology);
for (Entry<String, Object> entry : components.entrySet()) {
String componentName = entry.getKey();
Object component = entry.getValue();
ComponentCommon common = null;
if (component instanceof Bolt) {
common = ((Bolt) component).get_common();
}
if (component instanceof SpoutSpec) {
common = ((SpoutSpec) component).get_common();
}
if (component instanceof StateSpoutSpec) {
common = ((StateSpoutSpec) component).get_common();
}
String jsonConfString = common.get_json_conf();
if (jsonConfString == null) {
ret.put(componentName, context.DEFAULT_WEIGHT);
continue;
}
Map componentMap = new HashMap();
componentMap.putAll((Map) JStormUtils.from_json(jsonConfString));
int weight = computeWeight(componentMap, context);
ret.put(componentName, weight);
}
return ret;
}
private int computeWorkerNum() {
Integer settingNum = JStormUtils.parseInt(stormConf.get(Config.TOPOLOGY_WORKERS));
int hintSum = 0;
Map<String, Object> components = ThriftTopologyUtils.getComponents(sysTopology);
for (Entry<String, Object> entry : components.entrySet()) {
String componentName = entry.getKey();
Object component = entry.getValue();
ComponentCommon common = null;
if (component instanceof Bolt) {
common = ((Bolt) component).get_common();
}
if (component instanceof SpoutSpec) {
common = ((SpoutSpec) component).get_common();
}
if (component instanceof StateSpoutSpec) {
common = ((StateSpoutSpec) component).get_common();
}
int hint = common.get_parallelism_hint();
hintSum += hint;
}
if (settingNum == null) {
return hintSum;
} else {
return Math.min(settingNum, hintSum);
}
}
public IdDictionary(StormTopology topology) {
List<String> componentNames = new ArrayList<String>(topology.get_spouts().keySet());
componentNames.addAll(topology.get_bolts().keySet());
componentNames.addAll(topology.get_state_spouts().keySet());
for (String name : componentNames) {
ComponentCommon common = Utils.getComponentCommon(topology, name);
List<String> streams = new ArrayList<String>(common.get_streams().keySet());
streamNametoId.put(name, idify(streams));
streamIdToName.put(name, Utils.reverseMap(streamNametoId.get(name)));
}
}
@Override
public Map<Integer, ResourceAssignment> assignTasks(TopologyAssignContext context)
throws FailedAssignTopologyException {
int assignType = context.getAssignType();
if (TopologyAssignContext.isAssignTypeValid(assignType) == false) {
throw new FailedAssignTopologyException("Invalide Assign Type " + assignType);
}
DefaultTopologyAssignContext defaultContext = new DefaultTopologyAssignContext(context);
if (assignType == TopologyAssignContext.ASSIGN_TYPE_REBALANCE) {
freeUsed(defaultContext);
}
LOG.info("Dead tasks:" + defaultContext.getDeadTaskIds());
LOG.info("Unstopped tasks:" + defaultContext.getUnstoppedTaskIds());
Set<Integer> needAssignTasks = getNeedAssignTasks(defaultContext);
Map<Integer, ResourceAssignment> keepAssigns = getKeepAssign(defaultContext, needAssignTasks);
// please use tree map to make task sequence
Map<Integer, ResourceAssignment> ret = new TreeMap<Integer, ResourceAssignment>();
ret.putAll(keepAssigns);
ret.putAll(defaultContext.getUnstoppedAssignments());
Map<WorkerSlot, List<Integer>> keepAssignWorkers = Assignment.getWorkerTasks(keepAssigns);
int allocWorkerNum =
defaultContext.getTotalWorkerNum()
- defaultContext.getUnstoppedWorkerNum()
- keepAssignWorkers.size();
if (allocWorkerNum <= 0) {
LOG.warn(
"Don't need assign workers, all workers are fine " + defaultContext.toDetailString());
throw new FailedAssignTopologyException("Don't need assign worker, all workers are fine ");
}
Set<String> outputConfigComponents = new HashSet<String>();
Map<ComponentAssignType, Pair<Set<Integer>, IPreassignTask>> typeHandler =
registerPreAssignHandler(defaultContext, needAssignTasks);
Map<Integer, ResourceAssignment> newAssigns = new HashMap<Integer, ResourceAssignment>();
Set<String> usedSupervisorIds = new HashSet<String>();
List<Integer> lastFailed = new ArrayList<Integer>();
for (Entry<ComponentAssignType, Pair<Set<Integer>, IPreassignTask>> entry :
typeHandler.entrySet()) {
ComponentAssignType type = entry.getKey();
Set<Integer> tasks = entry.getValue().getFirst();
IPreassignTask handler = entry.getValue().getSecond();
tasks.addAll(lastFailed);
lastFailed.clear();
List<Integer> sortedTasks = sortAssignTasks(defaultContext, tasks);
StormTopology sysTopology = defaultContext.getSysTopology();
for (Integer task : sortedTasks) {
Set<String> canUsedSupervisorIds =
getCanUsedSupervisors(defaultContext, usedSupervisorIds, allocWorkerNum);
String componentName = defaultContext.getTaskToComponent().get(task);
ComponentCommon componentCommon =
ThriftTopologyUtils.getComponentCommon(sysTopology, componentName);
Map componentMap = (Map) JStormUtils.from_json(componentCommon.get_json_conf());
if (componentMap == null) {
componentMap = Maps.newHashMap();
}
if (outputConfigComponents.contains(componentName) == false) {
LOG.info("Component map of " + componentName + "\n" + componentMap);
outputConfigComponents.add(componentName);
}
ResourceAssignment preAssignment =
handler.preAssign(
task,
defaultContext,
componentMap,
componentName,
canUsedSupervisorIds,
ret,
newAssigns);
if (preAssignment == null) {
// pre assign fail
lastFailed.add(task);
} else {
// sucess to do preAssign
SupervisorInfo supervisorInfo =
defaultContext.getCluster().get(preAssignment.getSupervisorId());
LOG.info("Task " + task + " had been assigned to " + supervisorInfo.getHostName());
newAssigns.put(task, preAssignment);
ret.put(task, preAssignment);
usedSupervisorIds.add(preAssignment.getSupervisorId());
}
}
}
if (lastFailed.isEmpty() == false) {
throw new FailedAssignTopologyException("Failed to assign tasks " + lastFailed);
}
// Here just hardcode
IPostAssignTask postAssignHandler = new PostAssignTaskPort();
postAssignHandler.postAssign(defaultContext, newAssigns, allocWorkerNum);
LOG.info("Keep Alive slots:" + keepAssigns);
LOG.info("Unstopped slots:" + defaultContext.getUnstoppedAssignments());
LOG.info("New assign slots:" + newAssigns);
return ret;
}
/** Creates a Flink program that uses the specified spouts and bolts. */
private void translateTopology() {
unprocessdInputsPerBolt.clear();
outputStreams.clear();
declarers.clear();
availableInputs.clear();
// Storm defaults to parallelism 1
env.setParallelism(1);
/* Translation of topology */
for (final Entry<String, IRichSpout> spout : spouts.entrySet()) {
final String spoutId = spout.getKey();
final IRichSpout userSpout = spout.getValue();
final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer();
userSpout.declareOutputFields(declarer);
final HashMap<String, Fields> sourceStreams = declarer.outputStreams;
this.outputStreams.put(spoutId, sourceStreams);
declarers.put(spoutId, declarer);
final HashMap<String, DataStream<Tuple>> outputStreams =
new HashMap<String, DataStream<Tuple>>();
final DataStreamSource<?> source;
if (sourceStreams.size() == 1) {
final SpoutWrapper<Tuple> spoutWrapperSingleOutput =
new SpoutWrapper<Tuple>(userSpout, spoutId, null, null);
spoutWrapperSingleOutput.setStormTopology(stormTopology);
final String outputStreamId = (String) sourceStreams.keySet().toArray()[0];
DataStreamSource<Tuple> src =
env.addSource(
spoutWrapperSingleOutput, spoutId, declarer.getOutputType(outputStreamId));
outputStreams.put(outputStreamId, src);
source = src;
} else {
final SpoutWrapper<SplitStreamType<Tuple>> spoutWrapperMultipleOutputs =
new SpoutWrapper<SplitStreamType<Tuple>>(userSpout, spoutId, null, null);
spoutWrapperMultipleOutputs.setStormTopology(stormTopology);
@SuppressWarnings({"unchecked", "rawtypes"})
DataStreamSource<SplitStreamType<Tuple>> multiSource =
env.addSource(
spoutWrapperMultipleOutputs,
spoutId,
(TypeInformation) TypeExtractor.getForClass(SplitStreamType.class));
SplitStream<SplitStreamType<Tuple>> splitSource =
multiSource.split(new StormStreamSelector<Tuple>());
for (String streamId : sourceStreams.keySet()) {
SingleOutputStreamOperator<Tuple, ?> outStream =
splitSource.select(streamId).map(new SplitStreamMapper<Tuple>());
outStream.getTransformation().setOutputType(declarer.getOutputType(streamId));
outputStreams.put(streamId, outStream);
}
source = multiSource;
}
availableInputs.put(spoutId, outputStreams);
final ComponentCommon common = stormTopology.get_spouts().get(spoutId).get_common();
if (common.is_set_parallelism_hint()) {
int dop = common.get_parallelism_hint();
source.setParallelism(dop);
} else {
common.set_parallelism_hint(1);
}
}
/**
* 1. Connect all spout streams with bolts streams 2. Then proceed with the bolts stream already
* connected
*
* <p>Because we do not know the order in which an iterator steps over a set, we might process a
* consumer before its producer ->thus, we might need to repeat multiple times
*/
boolean makeProgress = true;
while (bolts.size() > 0) {
if (!makeProgress) {
StringBuilder strBld = new StringBuilder();
strBld.append("Unable to build Topology. Could not connect the following bolts:");
for (String boltId : bolts.keySet()) {
strBld.append("\n ");
strBld.append(boltId);
strBld.append(": missing input streams [");
for (Entry<GlobalStreamId, Grouping> streams : unprocessdInputsPerBolt.get(boltId)) {
strBld.append("'");
strBld.append(streams.getKey().get_streamId());
strBld.append("' from '");
strBld.append(streams.getKey().get_componentId());
strBld.append("'; ");
}
strBld.append("]");
}
throw new RuntimeException(strBld.toString());
}
makeProgress = false;
final Iterator<Entry<String, IRichBolt>> boltsIterator = bolts.entrySet().iterator();
while (boltsIterator.hasNext()) {
final Entry<String, IRichBolt> bolt = boltsIterator.next();
final String boltId = bolt.getKey();
final IRichBolt userBolt = copyObject(bolt.getValue());
final ComponentCommon common = stormTopology.get_bolts().get(boltId).get_common();
Set<Entry<GlobalStreamId, Grouping>> unprocessedBoltInputs =
unprocessdInputsPerBolt.get(boltId);
if (unprocessedBoltInputs == null) {
unprocessedBoltInputs = new HashSet<>();
unprocessedBoltInputs.addAll(common.get_inputs().entrySet());
unprocessdInputsPerBolt.put(boltId, unprocessedBoltInputs);
}
// check if all inputs are available
final int numberOfInputs = unprocessedBoltInputs.size();
int inputsAvailable = 0;
for (Entry<GlobalStreamId, Grouping> entry : unprocessedBoltInputs) {
final String producerId = entry.getKey().get_componentId();
final String streamId = entry.getKey().get_streamId();
final HashMap<String, DataStream<Tuple>> streams = availableInputs.get(producerId);
if (streams != null && streams.get(streamId) != null) {
inputsAvailable++;
}
}
if (inputsAvailable != numberOfInputs) {
// traverse other bolts first until inputs are available
continue;
} else {
makeProgress = true;
boltsIterator.remove();
}
final Map<GlobalStreamId, DataStream<Tuple>> inputStreams = new HashMap<>(numberOfInputs);
for (Entry<GlobalStreamId, Grouping> input : unprocessedBoltInputs) {
final GlobalStreamId streamId = input.getKey();
final Grouping grouping = input.getValue();
final String producerId = streamId.get_componentId();
final Map<String, DataStream<Tuple>> producer = availableInputs.get(producerId);
inputStreams.put(streamId, processInput(boltId, userBolt, streamId, grouping, producer));
}
final SingleOutputStreamOperator<?, ?> outputStream =
createOutput(boltId, userBolt, inputStreams);
if (common.is_set_parallelism_hint()) {
int dop = common.get_parallelism_hint();
outputStream.setParallelism(dop);
} else {
common.set_parallelism_hint(1);
}
}
}
}
