public static ComponentCommon getComponentCommon(StormTopology topology, String id) { if (topology.get_spouts().containsKey(id)) { return topology.get_spouts().get(id).get_common(); } if (topology.get_bolts().containsKey(id)) { return topology.get_bolts().get(id).get_common(); } if (topology.get_state_spouts().containsKey(id)) { return topology.get_state_spouts().get(id).get_common(); } throw new IllegalArgumentException("Could not find component with id " + id); }
public static Map<String, Map<String, Double>> getBoltsResources( StormTopology topology, Map topologyConf) { Map<String, Map<String, Double>> boltResources = new HashMap<String, Map<String, Double>>(); if (topology.get_bolts() != null) { for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) { Map<String, Double> topology_resources = parseResources(bolt.getValue().get_common().get_json_conf()); checkIntialization(topology_resources, bolt.getValue().toString(), topologyConf); boltResources.put(bolt.getKey(), topology_resources); } } return boltResources; }
private int setBoltInfo( StormTopology oldTopology, StormTopology newTopology, int cnt, StormClusterState clusterState) throws Exception { Map<String, Bolt> oldBolts = oldTopology.get_bolts(); Map<String, Bolt> bolts = newTopology.get_bolts(); for (Entry<String, Bolt> entry : oldBolts.entrySet()) { String boltName = entry.getKey(); Bolt oldBolt = entry.getValue(); Bolt bolt = bolts.get(boltName); if (oldBolt.get_common().get_parallelism_hint() > bolt.get_common().get_parallelism_hint()) { int removedTaskNum = oldBolt.get_common().get_parallelism_hint() - bolt.get_common().get_parallelism_hint(); TreeSet<Integer> taskIds = new TreeSet<Integer>(clusterState.task_ids_by_componentId(topologyid, boltName)); Iterator<Integer> descendIterator = taskIds.descendingIterator(); while (--removedTaskNum >= 0) { int taskId = descendIterator.next(); removeTask(topologyid, taskId, clusterState); LOG.info("Remove bolt task, taskId=" + taskId + " for " + boltName); } } else if (oldBolt.get_common().get_parallelism_hint() == bolt.get_common().get_parallelism_hint()) { continue; } else { int delta = bolt.get_common().get_parallelism_hint() - oldBolt.get_common().get_parallelism_hint(); Map<Integer, TaskInfo> taskInfoMap = new HashMap<Integer, TaskInfo>(); for (int i = 1; i <= delta; i++) { cnt++; TaskInfo taskInfo = new TaskInfo((String) entry.getKey(), "bolt"); taskInfoMap.put(cnt, taskInfo); newTasks.add(cnt); LOG.info("Setup new bolt task, taskId=" + cnt + " for " + boltName); } clusterState.add_task(topologyid, taskInfoMap); } } return cnt; }
public IdDictionary(StormTopology topology) { List<String> componentNames = new ArrayList<String>(topology.get_spouts().keySet()); componentNames.addAll(topology.get_bolts().keySet()); componentNames.addAll(topology.get_state_spouts().keySet()); for (String name : componentNames) { ComponentCommon common = Utils.getComponentCommon(topology, name); List<String> streams = new ArrayList<String>(common.get_streams().keySet()); streamNametoId.put(name, idify(streams)); streamIdToName.put(name, Utils.reverseMap(streamNametoId.get(name))); } }
/** * generate a taskid(Integer) for every task * * @param conf * @param topologyid * @return Map<Integer, String>: from taskid to componentid * @throws IOException * @throws InvalidTopologyException */ public Map<Integer, String> mkTaskComponentAssignments( Map<Object, Object> conf, String topologyid) throws IOException, InvalidTopologyException { // @@@ here exist a little problem, // we can directly pass stormConf from Submit method Map<Object, Object> stormConf = StormConfig.read_nimbus_topology_conf(conf, topologyid); StormTopology stopology = StormConfig.read_nimbus_topology_code(conf, topologyid); // use TreeMap to make task as sequence Map<Integer, String> rtn = new TreeMap<Integer, String>(); StormTopology topology = Common.system_topology(stormConf, stopology); Integer count = 0; count = mkTaskMaker(stormConf, topology.get_bolts(), rtn, count); count = mkTaskMaker(stormConf, topology.get_spouts(), rtn, count); count = mkTaskMaker(stormConf, topology.get_state_spouts(), rtn, count); return rtn; }
/** Creates a Flink program that uses the specified spouts and bolts. */ private void translateTopology() { unprocessdInputsPerBolt.clear(); outputStreams.clear(); declarers.clear(); availableInputs.clear(); // Storm defaults to parallelism 1 env.setParallelism(1); /* Translation of topology */ for (final Entry<String, IRichSpout> spout : spouts.entrySet()) { final String spoutId = spout.getKey(); final IRichSpout userSpout = spout.getValue(); final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer(); userSpout.declareOutputFields(declarer); final HashMap<String, Fields> sourceStreams = declarer.outputStreams; this.outputStreams.put(spoutId, sourceStreams); declarers.put(spoutId, declarer); final HashMap<String, DataStream<Tuple>> outputStreams = new HashMap<String, DataStream<Tuple>>(); final DataStreamSource<?> source; if (sourceStreams.size() == 1) { final SpoutWrapper<Tuple> spoutWrapperSingleOutput = new SpoutWrapper<Tuple>(userSpout, spoutId, null, null); spoutWrapperSingleOutput.setStormTopology(stormTopology); final String outputStreamId = (String) sourceStreams.keySet().toArray()[0]; DataStreamSource<Tuple> src = env.addSource( spoutWrapperSingleOutput, spoutId, declarer.getOutputType(outputStreamId)); outputStreams.put(outputStreamId, src); source = src; } else { final SpoutWrapper<SplitStreamType<Tuple>> spoutWrapperMultipleOutputs = new SpoutWrapper<SplitStreamType<Tuple>>(userSpout, spoutId, null, null); spoutWrapperMultipleOutputs.setStormTopology(stormTopology); @SuppressWarnings({"unchecked", "rawtypes"}) DataStreamSource<SplitStreamType<Tuple>> multiSource = env.addSource( spoutWrapperMultipleOutputs, spoutId, (TypeInformation) TypeExtractor.getForClass(SplitStreamType.class)); SplitStream<SplitStreamType<Tuple>> splitSource = multiSource.split(new StormStreamSelector<Tuple>()); for (String streamId : sourceStreams.keySet()) { SingleOutputStreamOperator<Tuple, ?> outStream = splitSource.select(streamId).map(new SplitStreamMapper<Tuple>()); outStream.getTransformation().setOutputType(declarer.getOutputType(streamId)); outputStreams.put(streamId, outStream); } source = multiSource; } availableInputs.put(spoutId, outputStreams); final ComponentCommon common = stormTopology.get_spouts().get(spoutId).get_common(); if (common.is_set_parallelism_hint()) { int dop = common.get_parallelism_hint(); source.setParallelism(dop); } else { common.set_parallelism_hint(1); } } /** * 1. Connect all spout streams with bolts streams 2. Then proceed with the bolts stream already * connected * * <p>Because we do not know the order in which an iterator steps over a set, we might process a * consumer before its producer ->thus, we might need to repeat multiple times */ boolean makeProgress = true; while (bolts.size() > 0) { if (!makeProgress) { StringBuilder strBld = new StringBuilder(); strBld.append("Unable to build Topology. Could not connect the following bolts:"); for (String boltId : bolts.keySet()) { strBld.append("\n "); strBld.append(boltId); strBld.append(": missing input streams ["); for (Entry<GlobalStreamId, Grouping> streams : unprocessdInputsPerBolt.get(boltId)) { strBld.append("'"); strBld.append(streams.getKey().get_streamId()); strBld.append("' from '"); strBld.append(streams.getKey().get_componentId()); strBld.append("'; "); } strBld.append("]"); } throw new RuntimeException(strBld.toString()); } makeProgress = false; final Iterator<Entry<String, IRichBolt>> boltsIterator = bolts.entrySet().iterator(); while (boltsIterator.hasNext()) { final Entry<String, IRichBolt> bolt = boltsIterator.next(); final String boltId = bolt.getKey(); final IRichBolt userBolt = copyObject(bolt.getValue()); final ComponentCommon common = stormTopology.get_bolts().get(boltId).get_common(); Set<Entry<GlobalStreamId, Grouping>> unprocessedBoltInputs = unprocessdInputsPerBolt.get(boltId); if (unprocessedBoltInputs == null) { unprocessedBoltInputs = new HashSet<>(); unprocessedBoltInputs.addAll(common.get_inputs().entrySet()); unprocessdInputsPerBolt.put(boltId, unprocessedBoltInputs); } // check if all inputs are available final int numberOfInputs = unprocessedBoltInputs.size(); int inputsAvailable = 0; for (Entry<GlobalStreamId, Grouping> entry : unprocessedBoltInputs) { final String producerId = entry.getKey().get_componentId(); final String streamId = entry.getKey().get_streamId(); final HashMap<String, DataStream<Tuple>> streams = availableInputs.get(producerId); if (streams != null && streams.get(streamId) != null) { inputsAvailable++; } } if (inputsAvailable != numberOfInputs) { // traverse other bolts first until inputs are available continue; } else { makeProgress = true; boltsIterator.remove(); } final Map<GlobalStreamId, DataStream<Tuple>> inputStreams = new HashMap<>(numberOfInputs); for (Entry<GlobalStreamId, Grouping> input : unprocessedBoltInputs) { final GlobalStreamId streamId = input.getKey(); final Grouping grouping = input.getValue(); final String producerId = streamId.get_componentId(); final Map<String, DataStream<Tuple>> producer = availableInputs.get(producerId); inputStreams.put(streamId, processInput(boltId, userBolt, streamId, grouping, producer)); } final SingleOutputStreamOperator<?, ?> outputStream = createOutput(boltId, userBolt, inputStreams); if (common.is_set_parallelism_hint()) { int dop = common.get_parallelism_hint(); outputStream.setParallelism(dop); } else { common.set_parallelism_hint(1); } } } }
/** * Returns a representation of the non-system components of the topology graph Each Component * object in the returning map is populated with the list of its parents, children and execs * assigned to that component. * * @return a map of components */ public Map<String, Component> getComponents() { Map<String, Component> all_comp = new HashMap<String, Component>(); StormTopology storm_topo = this.topology; // spouts if (storm_topo.get_spouts() != null) { for (Map.Entry<String, SpoutSpec> spoutEntry : storm_topo.get_spouts().entrySet()) { if (!Utils.isSystemId(spoutEntry.getKey())) { Component newComp = null; if (all_comp.containsKey(spoutEntry.getKey())) { newComp = all_comp.get(spoutEntry.getKey()); newComp.execs = componentToExecs(newComp.id); } else { newComp = new Component(spoutEntry.getKey()); newComp.execs = componentToExecs(newComp.id); all_comp.put(spoutEntry.getKey(), newComp); } newComp.type = Component.ComponentType.SPOUT; for (Map.Entry<GlobalStreamId, Grouping> spoutInput : spoutEntry.getValue().get_common().get_inputs().entrySet()) { newComp.parents.add(spoutInput.getKey().get_componentId()); if (!all_comp.containsKey(spoutInput.getKey().get_componentId())) { all_comp.put( spoutInput.getKey().get_componentId(), new Component(spoutInput.getKey().get_componentId())); } all_comp.get(spoutInput.getKey().get_componentId()).children.add(spoutEntry.getKey()); } } } } // bolts if (storm_topo.get_bolts() != null) { for (Map.Entry<String, Bolt> boltEntry : storm_topo.get_bolts().entrySet()) { if (!Utils.isSystemId(boltEntry.getKey())) { Component newComp = null; if (all_comp.containsKey(boltEntry.getKey())) { newComp = all_comp.get(boltEntry.getKey()); newComp.execs = componentToExecs(newComp.id); } else { newComp = new Component(boltEntry.getKey()); newComp.execs = componentToExecs(newComp.id); all_comp.put(boltEntry.getKey(), newComp); } newComp.type = Component.ComponentType.BOLT; for (Map.Entry<GlobalStreamId, Grouping> boltInput : boltEntry.getValue().get_common().get_inputs().entrySet()) { newComp.parents.add(boltInput.getKey().get_componentId()); if (!all_comp.containsKey(boltInput.getKey().get_componentId())) { all_comp.put( boltInput.getKey().get_componentId(), new Component(boltInput.getKey().get_componentId())); } all_comp.get(boltInput.getKey().get_componentId()).children.add(boltEntry.getKey()); } } } } return all_comp; }