public IdDictionary(StormTopology topology) {
List<String> componentNames = new ArrayList<String>(topology.get_spouts().keySet());
componentNames.addAll(topology.get_bolts().keySet());
componentNames.addAll(topology.get_state_spouts().keySet());
for (String name : componentNames) {
ComponentCommon common = Utils.getComponentCommon(topology, name);
List<String> streams = new ArrayList<String>(common.get_streams().keySet());
streamNametoId.put(name, idify(streams));
streamIdToName.put(name, Utils.reverseMap(streamNametoId.get(name)));
}
}
public static Map<String, Map<String, Double>> getSpoutsResources(
StormTopology topology, Map topologyConf) {
Map<String, Map<String, Double>> spoutResources = new HashMap<String, Map<String, Double>>();
if (topology.get_spouts() != null) {
for (Map.Entry<String, SpoutSpec> spout : topology.get_spouts().entrySet()) {
Map<String, Double> topology_resources =
parseResources(spout.getValue().get_common().get_json_conf());
checkIntialization(topology_resources, spout.getValue().toString(), topologyConf);
spoutResources.put(spout.getKey(), topology_resources);
}
}
return spoutResources;
}
public static Map<String, Map<String, Double>> getBoltsResources(
StormTopology topology, Map topologyConf) {
Map<String, Map<String, Double>> boltResources = new HashMap<String, Map<String, Double>>();
if (topology.get_bolts() != null) {
for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) {
Map<String, Double> topology_resources =
parseResources(bolt.getValue().get_common().get_json_conf());
checkIntialization(topology_resources, bolt.getValue().toString(), topologyConf);
boltResources.put(bolt.getKey(), topology_resources);
}
}
return boltResources;
}
private int setSpoutInfo(
StormTopology oldTopology, StormTopology newTopology, int cnt, StormClusterState clusterState)
throws Exception {
Map<String, SpoutSpec> oldSpouts = oldTopology.get_spouts();
Map<String, SpoutSpec> spouts = newTopology.get_spouts();
for (Entry<String, SpoutSpec> entry : oldSpouts.entrySet()) {
String spoutName = entry.getKey();
SpoutSpec oldSpout = entry.getValue();
SpoutSpec spout = spouts.get(spoutName);
if (oldSpout.get_common().get_parallelism_hint()
> spout.get_common().get_parallelism_hint()) {
int removedTaskNum =
oldSpout.get_common().get_parallelism_hint()
- spout.get_common().get_parallelism_hint();
TreeSet<Integer> taskIds =
new TreeSet<Integer>(clusterState.task_ids_by_componentId(topologyid, spoutName));
Iterator<Integer> descendIterator = taskIds.descendingIterator();
while (--removedTaskNum >= 0) {
int taskId = descendIterator.next();
removeTask(topologyid, taskId, clusterState);
LOG.info("Remove spout task, taskId=" + taskId + " for " + spoutName);
}
} else if (oldSpout.get_common().get_parallelism_hint()
== spout.get_common().get_parallelism_hint()) {
continue;
} else {
int delta =
spout.get_common().get_parallelism_hint()
- oldSpout.get_common().get_parallelism_hint();
Map<Integer, TaskInfo> taskInfoMap = new HashMap<Integer, TaskInfo>();
for (int i = 1; i <= delta; i++) {
cnt++;
TaskInfo taskInfo = new TaskInfo((String) entry.getKey(), "spout");
taskInfoMap.put(cnt, taskInfo);
newTasks.add(cnt);
LOG.info("Setup new spout task, taskId=" + cnt + " for " + spoutName);
}
clusterState.add_task(topologyid, taskInfoMap);
}
}
return cnt;
}
/**
* generate a taskid(Integer) for every task
*
* @param conf
* @param topologyid
* @return Map<Integer, String>: from taskid to componentid
* @throws IOException
* @throws InvalidTopologyException
*/
public Map<Integer, String> mkTaskComponentAssignments(
Map<Object, Object> conf, String topologyid) throws IOException, InvalidTopologyException {
// @@@ here exist a little problem,
// we can directly pass stormConf from Submit method
Map<Object, Object> stormConf = StormConfig.read_nimbus_topology_conf(conf, topologyid);
StormTopology stopology = StormConfig.read_nimbus_topology_code(conf, topologyid);
// use TreeMap to make task as sequence
Map<Integer, String> rtn = new TreeMap<Integer, String>();
StormTopology topology = Common.system_topology(stormConf, stopology);
Integer count = 0;
count = mkTaskMaker(stormConf, topology.get_bolts(), rtn, count);
count = mkTaskMaker(stormConf, topology.get_spouts(), rtn, count);
count = mkTaskMaker(stormConf, topology.get_state_spouts(), rtn, count);
return rtn;
}
public static ComponentCommon getComponentCommon(StormTopology topology, String id) {
if (topology.get_spouts().containsKey(id)) {
return topology.get_spouts().get(id).get_common();
}
if (topology.get_bolts().containsKey(id)) {
return topology.get_bolts().get(id).get_common();
}
if (topology.get_state_spouts().containsKey(id)) {
return topology.get_state_spouts().get(id).get_common();
}
throw new IllegalArgumentException("Could not find component with id " + id);
}
@Override
public <T> Object execute(T... args) {
boolean isSetTaskInfo = false;
try {
Boolean reassign = (Boolean) args[1];
Map<Object, Object> conf = (Map<Object, Object>) args[2]; // args[0]:
// delay,
// args[1]:
// reassign_flag,
// args[2]:
// conf
if (conf != null) {
boolean isConfUpdate = false;
Map stormConf = data.getConf();
// Update topology code
Map topoConf = StormConfig.read_nimbus_topology_conf(stormConf, topologyid);
StormTopology rawOldTopology = StormConfig.read_nimbus_topology_code(stormConf, topologyid);
StormTopology rawNewTopology = NimbusUtils.normalizeTopology(conf, rawOldTopology, true);
StormTopology sysOldTopology = rawOldTopology.deepCopy();
StormTopology sysNewTopology = rawNewTopology.deepCopy();
if (conf.get(Config.TOPOLOGY_ACKER_EXECUTORS) != null) {
Common.add_acker(topoConf, sysOldTopology);
Common.add_acker(conf, sysNewTopology);
int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
int oldAckerNum = JStormUtils.parseInt(topoConf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
LOG.info("Update acker from oldAckerNum=" + oldAckerNum + " to ackerNum=" + ackerNum);
topoConf.put(Config.TOPOLOGY_ACKER_EXECUTORS, ackerNum);
isConfUpdate = true;
}
// If scale-out, setup task info for new added tasks
setTaskInfo(sysOldTopology, sysNewTopology);
isSetTaskInfo = true;
// If everything is OK, write topology code into disk
StormConfig.write_nimbus_topology_code(
stormConf, topologyid, Utils.serialize(rawNewTopology));
// Update topology conf if worker num has been updated
Set<Object> keys = conf.keySet();
Integer workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS));
if (workerNum != null) {
Integer oldWorkerNum = JStormUtils.parseInt(topoConf.get(Config.TOPOLOGY_WORKERS));
topoConf.put(Config.TOPOLOGY_WORKERS, workerNum);
isConfUpdate = true;
LOG.info("Update worker num from " + oldWorkerNum + " to " + workerNum);
}
if (keys.contains(Config.ISOLATION_SCHEDULER_MACHINES)) {
topoConf.put(
Config.ISOLATION_SCHEDULER_MACHINES, conf.get(Config.ISOLATION_SCHEDULER_MACHINES));
}
if (isConfUpdate) {
StormConfig.write_nimbus_topology_conf(stormConf, topologyid, topoConf);
}
}
TopologyAssignEvent event = new TopologyAssignEvent();
event.setTopologyId(topologyid);
event.setScratch(true);
event.setOldStatus(oldStatus);
event.setReassign(reassign);
if (conf != null) event.setScaleTopology(true);
TopologyAssign.push(event);
event.waitFinish();
} catch (Exception e) {
LOG.error("do-rebalance error!", e);
// Rollback the changes on ZK
if (isSetTaskInfo) {
try {
StormClusterState clusterState = data.getStormClusterState();
clusterState.remove_task(topologyid, newTasks);
} catch (Exception e1) {
LOG.error("Failed to rollback the changes on ZK for task-" + newTasks, e);
}
}
}
DelayStatusTransitionCallback delayCallback =
new DelayStatusTransitionCallback(
data, topologyid, oldStatus, StatusType.rebalancing, StatusType.done_rebalance);
return delayCallback.execute();
}
/** Creates a Flink program that uses the specified spouts and bolts. */
private void translateTopology() {
unprocessdInputsPerBolt.clear();
outputStreams.clear();
declarers.clear();
availableInputs.clear();
// Storm defaults to parallelism 1
env.setParallelism(1);
/* Translation of topology */
for (final Entry<String, IRichSpout> spout : spouts.entrySet()) {
final String spoutId = spout.getKey();
final IRichSpout userSpout = spout.getValue();
final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer();
userSpout.declareOutputFields(declarer);
final HashMap<String, Fields> sourceStreams = declarer.outputStreams;
this.outputStreams.put(spoutId, sourceStreams);
declarers.put(spoutId, declarer);
final HashMap<String, DataStream<Tuple>> outputStreams =
new HashMap<String, DataStream<Tuple>>();
final DataStreamSource<?> source;
if (sourceStreams.size() == 1) {
final SpoutWrapper<Tuple> spoutWrapperSingleOutput =
new SpoutWrapper<Tuple>(userSpout, spoutId, null, null);
spoutWrapperSingleOutput.setStormTopology(stormTopology);
final String outputStreamId = (String) sourceStreams.keySet().toArray()[0];
DataStreamSource<Tuple> src =
env.addSource(
spoutWrapperSingleOutput, spoutId, declarer.getOutputType(outputStreamId));
outputStreams.put(outputStreamId, src);
source = src;
} else {
final SpoutWrapper<SplitStreamType<Tuple>> spoutWrapperMultipleOutputs =
new SpoutWrapper<SplitStreamType<Tuple>>(userSpout, spoutId, null, null);
spoutWrapperMultipleOutputs.setStormTopology(stormTopology);
@SuppressWarnings({"unchecked", "rawtypes"})
DataStreamSource<SplitStreamType<Tuple>> multiSource =
env.addSource(
spoutWrapperMultipleOutputs,
spoutId,
(TypeInformation) TypeExtractor.getForClass(SplitStreamType.class));
SplitStream<SplitStreamType<Tuple>> splitSource =
multiSource.split(new StormStreamSelector<Tuple>());
for (String streamId : sourceStreams.keySet()) {
SingleOutputStreamOperator<Tuple, ?> outStream =
splitSource.select(streamId).map(new SplitStreamMapper<Tuple>());
outStream.getTransformation().setOutputType(declarer.getOutputType(streamId));
outputStreams.put(streamId, outStream);
}
source = multiSource;
}
availableInputs.put(spoutId, outputStreams);
final ComponentCommon common = stormTopology.get_spouts().get(spoutId).get_common();
if (common.is_set_parallelism_hint()) {
int dop = common.get_parallelism_hint();
source.setParallelism(dop);
} else {
common.set_parallelism_hint(1);
}
}
/**
* 1. Connect all spout streams with bolts streams 2. Then proceed with the bolts stream already
* connected
*
* <p>Because we do not know the order in which an iterator steps over a set, we might process a
* consumer before its producer ->thus, we might need to repeat multiple times
*/
boolean makeProgress = true;
while (bolts.size() > 0) {
if (!makeProgress) {
StringBuilder strBld = new StringBuilder();
strBld.append("Unable to build Topology. Could not connect the following bolts:");
for (String boltId : bolts.keySet()) {
strBld.append("\n ");
strBld.append(boltId);
strBld.append(": missing input streams [");
for (Entry<GlobalStreamId, Grouping> streams : unprocessdInputsPerBolt.get(boltId)) {
strBld.append("'");
strBld.append(streams.getKey().get_streamId());
strBld.append("' from '");
strBld.append(streams.getKey().get_componentId());
strBld.append("'; ");
}
strBld.append("]");
}
throw new RuntimeException(strBld.toString());
}
makeProgress = false;
final Iterator<Entry<String, IRichBolt>> boltsIterator = bolts.entrySet().iterator();
while (boltsIterator.hasNext()) {
final Entry<String, IRichBolt> bolt = boltsIterator.next();
final String boltId = bolt.getKey();
final IRichBolt userBolt = copyObject(bolt.getValue());
final ComponentCommon common = stormTopology.get_bolts().get(boltId).get_common();
Set<Entry<GlobalStreamId, Grouping>> unprocessedBoltInputs =
unprocessdInputsPerBolt.get(boltId);
if (unprocessedBoltInputs == null) {
unprocessedBoltInputs = new HashSet<>();
unprocessedBoltInputs.addAll(common.get_inputs().entrySet());
unprocessdInputsPerBolt.put(boltId, unprocessedBoltInputs);
}
// check if all inputs are available
final int numberOfInputs = unprocessedBoltInputs.size();
int inputsAvailable = 0;
for (Entry<GlobalStreamId, Grouping> entry : unprocessedBoltInputs) {
final String producerId = entry.getKey().get_componentId();
final String streamId = entry.getKey().get_streamId();
final HashMap<String, DataStream<Tuple>> streams = availableInputs.get(producerId);
if (streams != null && streams.get(streamId) != null) {
inputsAvailable++;
}
}
if (inputsAvailable != numberOfInputs) {
// traverse other bolts first until inputs are available
continue;
} else {
makeProgress = true;
boltsIterator.remove();
}
final Map<GlobalStreamId, DataStream<Tuple>> inputStreams = new HashMap<>(numberOfInputs);
for (Entry<GlobalStreamId, Grouping> input : unprocessedBoltInputs) {
final GlobalStreamId streamId = input.getKey();
final Grouping grouping = input.getValue();
final String producerId = streamId.get_componentId();
final Map<String, DataStream<Tuple>> producer = availableInputs.get(producerId);
inputStreams.put(streamId, processInput(boltId, userBolt, streamId, grouping, producer));
}
final SingleOutputStreamOperator<?, ?> outputStream =
createOutput(boltId, userBolt, inputStreams);
if (common.is_set_parallelism_hint()) {
int dop = common.get_parallelism_hint();
outputStream.setParallelism(dop);
} else {
common.set_parallelism_hint(1);
}
}
}
}
/**
* Returns a representation of the non-system components of the topology graph Each Component
* object in the returning map is populated with the list of its parents, children and execs
* assigned to that component.
*
* @return a map of components
*/
public Map<String, Component> getComponents() {
Map<String, Component> all_comp = new HashMap<String, Component>();
StormTopology storm_topo = this.topology;
// spouts
if (storm_topo.get_spouts() != null) {
for (Map.Entry<String, SpoutSpec> spoutEntry : storm_topo.get_spouts().entrySet()) {
if (!Utils.isSystemId(spoutEntry.getKey())) {
Component newComp = null;
if (all_comp.containsKey(spoutEntry.getKey())) {
newComp = all_comp.get(spoutEntry.getKey());
newComp.execs = componentToExecs(newComp.id);
} else {
newComp = new Component(spoutEntry.getKey());
newComp.execs = componentToExecs(newComp.id);
all_comp.put(spoutEntry.getKey(), newComp);
}
newComp.type = Component.ComponentType.SPOUT;
for (Map.Entry<GlobalStreamId, Grouping> spoutInput :
spoutEntry.getValue().get_common().get_inputs().entrySet()) {
newComp.parents.add(spoutInput.getKey().get_componentId());
if (!all_comp.containsKey(spoutInput.getKey().get_componentId())) {
all_comp.put(
spoutInput.getKey().get_componentId(),
new Component(spoutInput.getKey().get_componentId()));
}
all_comp.get(spoutInput.getKey().get_componentId()).children.add(spoutEntry.getKey());
}
}
}
}
// bolts
if (storm_topo.get_bolts() != null) {
for (Map.Entry<String, Bolt> boltEntry : storm_topo.get_bolts().entrySet()) {
if (!Utils.isSystemId(boltEntry.getKey())) {
Component newComp = null;
if (all_comp.containsKey(boltEntry.getKey())) {
newComp = all_comp.get(boltEntry.getKey());
newComp.execs = componentToExecs(newComp.id);
} else {
newComp = new Component(boltEntry.getKey());
newComp.execs = componentToExecs(newComp.id);
all_comp.put(boltEntry.getKey(), newComp);
}
newComp.type = Component.ComponentType.BOLT;
for (Map.Entry<GlobalStreamId, Grouping> boltInput :
boltEntry.getValue().get_common().get_inputs().entrySet()) {
newComp.parents.add(boltInput.getKey().get_componentId());
if (!all_comp.containsKey(boltInput.getKey().get_componentId())) {
all_comp.put(
boltInput.getKey().get_componentId(),
new Component(boltInput.getKey().get_componentId()));
}
all_comp.get(boltInput.getKey().get_componentId()).children.add(boltEntry.getKey());
}
}
}
}
return all_comp;
}
