Example #1
0
    public IdDictionary(StormTopology topology) {
      List<String> componentNames = new ArrayList<String>(topology.get_spouts().keySet());
      componentNames.addAll(topology.get_bolts().keySet());
      componentNames.addAll(topology.get_state_spouts().keySet());

      for (String name : componentNames) {
        ComponentCommon common = Utils.getComponentCommon(topology, name);
        List<String> streams = new ArrayList<String>(common.get_streams().keySet());
        streamNametoId.put(name, idify(streams));
        streamIdToName.put(name, Utils.reverseMap(streamNametoId.get(name)));
      }
    }
Example #2
0
 public static Map<String, Map<String, Double>> getSpoutsResources(
     StormTopology topology, Map topologyConf) {
   Map<String, Map<String, Double>> spoutResources = new HashMap<String, Map<String, Double>>();
   if (topology.get_spouts() != null) {
     for (Map.Entry<String, SpoutSpec> spout : topology.get_spouts().entrySet()) {
       Map<String, Double> topology_resources =
           parseResources(spout.getValue().get_common().get_json_conf());
       checkIntialization(topology_resources, spout.getValue().toString(), topologyConf);
       spoutResources.put(spout.getKey(), topology_resources);
     }
   }
   return spoutResources;
 }
Example #3
0
 public static Map<String, Map<String, Double>> getBoltsResources(
     StormTopology topology, Map topologyConf) {
   Map<String, Map<String, Double>> boltResources = new HashMap<String, Map<String, Double>>();
   if (topology.get_bolts() != null) {
     for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) {
       Map<String, Double> topology_resources =
           parseResources(bolt.getValue().get_common().get_json_conf());
       checkIntialization(topology_resources, bolt.getValue().toString(), topologyConf);
       boltResources.put(bolt.getKey(), topology_resources);
     }
   }
   return boltResources;
 }
  private int setSpoutInfo(
      StormTopology oldTopology, StormTopology newTopology, int cnt, StormClusterState clusterState)
      throws Exception {
    Map<String, SpoutSpec> oldSpouts = oldTopology.get_spouts();
    Map<String, SpoutSpec> spouts = newTopology.get_spouts();
    for (Entry<String, SpoutSpec> entry : oldSpouts.entrySet()) {
      String spoutName = entry.getKey();
      SpoutSpec oldSpout = entry.getValue();
      SpoutSpec spout = spouts.get(spoutName);
      if (oldSpout.get_common().get_parallelism_hint()
          > spout.get_common().get_parallelism_hint()) {
        int removedTaskNum =
            oldSpout.get_common().get_parallelism_hint()
                - spout.get_common().get_parallelism_hint();
        TreeSet<Integer> taskIds =
            new TreeSet<Integer>(clusterState.task_ids_by_componentId(topologyid, spoutName));
        Iterator<Integer> descendIterator = taskIds.descendingIterator();
        while (--removedTaskNum >= 0) {
          int taskId = descendIterator.next();
          removeTask(topologyid, taskId, clusterState);
          LOG.info("Remove spout task, taskId=" + taskId + " for " + spoutName);
        }

      } else if (oldSpout.get_common().get_parallelism_hint()
          == spout.get_common().get_parallelism_hint()) {
        continue;
      } else {
        int delta =
            spout.get_common().get_parallelism_hint()
                - oldSpout.get_common().get_parallelism_hint();
        Map<Integer, TaskInfo> taskInfoMap = new HashMap<Integer, TaskInfo>();

        for (int i = 1; i <= delta; i++) {
          cnt++;
          TaskInfo taskInfo = new TaskInfo((String) entry.getKey(), "spout");
          taskInfoMap.put(cnt, taskInfo);
          newTasks.add(cnt);
          LOG.info("Setup new spout task, taskId=" + cnt + " for " + spoutName);
        }
        clusterState.add_task(topologyid, taskInfoMap);
      }
    }

    return cnt;
  }
Example #5
0
  /**
   * generate a taskid(Integer) for every task
   *
   * @param conf
   * @param topologyid
   * @return Map<Integer, String>: from taskid to componentid
   * @throws IOException
   * @throws InvalidTopologyException
   */
  public Map<Integer, String> mkTaskComponentAssignments(
      Map<Object, Object> conf, String topologyid) throws IOException, InvalidTopologyException {

    // @@@ here exist a little problem,
    // we can directly pass stormConf from Submit method
    Map<Object, Object> stormConf = StormConfig.read_nimbus_topology_conf(conf, topologyid);

    StormTopology stopology = StormConfig.read_nimbus_topology_code(conf, topologyid);

    // use TreeMap to make task as sequence
    Map<Integer, String> rtn = new TreeMap<Integer, String>();

    StormTopology topology = Common.system_topology(stormConf, stopology);

    Integer count = 0;
    count = mkTaskMaker(stormConf, topology.get_bolts(), rtn, count);
    count = mkTaskMaker(stormConf, topology.get_spouts(), rtn, count);
    count = mkTaskMaker(stormConf, topology.get_state_spouts(), rtn, count);

    return rtn;
  }
Example #6
0
 public static ComponentCommon getComponentCommon(StormTopology topology, String id) {
   if (topology.get_spouts().containsKey(id)) {
     return topology.get_spouts().get(id).get_common();
   }
   if (topology.get_bolts().containsKey(id)) {
     return topology.get_bolts().get(id).get_common();
   }
   if (topology.get_state_spouts().containsKey(id)) {
     return topology.get_state_spouts().get(id).get_common();
   }
   throw new IllegalArgumentException("Could not find component with id " + id);
 }
  @Override
  public <T> Object execute(T... args) {
    boolean isSetTaskInfo = false;
    try {
      Boolean reassign = (Boolean) args[1];
      Map<Object, Object> conf = (Map<Object, Object>) args[2]; // args[0]:
      // delay,
      // args[1]:
      // reassign_flag,
      // args[2]:
      // conf
      if (conf != null) {
        boolean isConfUpdate = false;
        Map stormConf = data.getConf();

        // Update topology code
        Map topoConf = StormConfig.read_nimbus_topology_conf(stormConf, topologyid);
        StormTopology rawOldTopology = StormConfig.read_nimbus_topology_code(stormConf, topologyid);
        StormTopology rawNewTopology = NimbusUtils.normalizeTopology(conf, rawOldTopology, true);
        StormTopology sysOldTopology = rawOldTopology.deepCopy();
        StormTopology sysNewTopology = rawNewTopology.deepCopy();
        if (conf.get(Config.TOPOLOGY_ACKER_EXECUTORS) != null) {
          Common.add_acker(topoConf, sysOldTopology);
          Common.add_acker(conf, sysNewTopology);
          int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
          int oldAckerNum = JStormUtils.parseInt(topoConf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
          LOG.info("Update acker from oldAckerNum=" + oldAckerNum + " to ackerNum=" + ackerNum);
          topoConf.put(Config.TOPOLOGY_ACKER_EXECUTORS, ackerNum);
          isConfUpdate = true;
        }

        // If scale-out, setup task info for new added tasks
        setTaskInfo(sysOldTopology, sysNewTopology);
        isSetTaskInfo = true;

        // If everything is OK, write topology code into disk
        StormConfig.write_nimbus_topology_code(
            stormConf, topologyid, Utils.serialize(rawNewTopology));

        // Update topology conf if worker num has been updated
        Set<Object> keys = conf.keySet();
        Integer workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS));
        if (workerNum != null) {
          Integer oldWorkerNum = JStormUtils.parseInt(topoConf.get(Config.TOPOLOGY_WORKERS));
          topoConf.put(Config.TOPOLOGY_WORKERS, workerNum);
          isConfUpdate = true;

          LOG.info("Update worker num from " + oldWorkerNum + " to " + workerNum);
        }

        if (keys.contains(Config.ISOLATION_SCHEDULER_MACHINES)) {
          topoConf.put(
              Config.ISOLATION_SCHEDULER_MACHINES, conf.get(Config.ISOLATION_SCHEDULER_MACHINES));
        }

        if (isConfUpdate) {
          StormConfig.write_nimbus_topology_conf(stormConf, topologyid, topoConf);
        }
      }

      TopologyAssignEvent event = new TopologyAssignEvent();

      event.setTopologyId(topologyid);
      event.setScratch(true);
      event.setOldStatus(oldStatus);
      event.setReassign(reassign);
      if (conf != null) event.setScaleTopology(true);
      TopologyAssign.push(event);
      event.waitFinish();
    } catch (Exception e) {
      LOG.error("do-rebalance error!", e);
      // Rollback the changes on ZK
      if (isSetTaskInfo) {
        try {
          StormClusterState clusterState = data.getStormClusterState();
          clusterState.remove_task(topologyid, newTasks);
        } catch (Exception e1) {
          LOG.error("Failed to rollback the changes on ZK for task-" + newTasks, e);
        }
      }
    }

    DelayStatusTransitionCallback delayCallback =
        new DelayStatusTransitionCallback(
            data, topologyid, oldStatus, StatusType.rebalancing, StatusType.done_rebalance);
    return delayCallback.execute();
  }
Example #8
0
  /** Creates a Flink program that uses the specified spouts and bolts. */
  private void translateTopology() {

    unprocessdInputsPerBolt.clear();
    outputStreams.clear();
    declarers.clear();
    availableInputs.clear();

    // Storm defaults to parallelism 1
    env.setParallelism(1);

    /* Translation of topology */

    for (final Entry<String, IRichSpout> spout : spouts.entrySet()) {
      final String spoutId = spout.getKey();
      final IRichSpout userSpout = spout.getValue();

      final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer();
      userSpout.declareOutputFields(declarer);
      final HashMap<String, Fields> sourceStreams = declarer.outputStreams;
      this.outputStreams.put(spoutId, sourceStreams);
      declarers.put(spoutId, declarer);

      final HashMap<String, DataStream<Tuple>> outputStreams =
          new HashMap<String, DataStream<Tuple>>();
      final DataStreamSource<?> source;

      if (sourceStreams.size() == 1) {
        final SpoutWrapper<Tuple> spoutWrapperSingleOutput =
            new SpoutWrapper<Tuple>(userSpout, spoutId, null, null);
        spoutWrapperSingleOutput.setStormTopology(stormTopology);

        final String outputStreamId = (String) sourceStreams.keySet().toArray()[0];

        DataStreamSource<Tuple> src =
            env.addSource(
                spoutWrapperSingleOutput, spoutId, declarer.getOutputType(outputStreamId));

        outputStreams.put(outputStreamId, src);
        source = src;
      } else {
        final SpoutWrapper<SplitStreamType<Tuple>> spoutWrapperMultipleOutputs =
            new SpoutWrapper<SplitStreamType<Tuple>>(userSpout, spoutId, null, null);
        spoutWrapperMultipleOutputs.setStormTopology(stormTopology);

        @SuppressWarnings({"unchecked", "rawtypes"})
        DataStreamSource<SplitStreamType<Tuple>> multiSource =
            env.addSource(
                spoutWrapperMultipleOutputs,
                spoutId,
                (TypeInformation) TypeExtractor.getForClass(SplitStreamType.class));

        SplitStream<SplitStreamType<Tuple>> splitSource =
            multiSource.split(new StormStreamSelector<Tuple>());
        for (String streamId : sourceStreams.keySet()) {
          SingleOutputStreamOperator<Tuple, ?> outStream =
              splitSource.select(streamId).map(new SplitStreamMapper<Tuple>());
          outStream.getTransformation().setOutputType(declarer.getOutputType(streamId));
          outputStreams.put(streamId, outStream);
        }
        source = multiSource;
      }
      availableInputs.put(spoutId, outputStreams);

      final ComponentCommon common = stormTopology.get_spouts().get(spoutId).get_common();
      if (common.is_set_parallelism_hint()) {
        int dop = common.get_parallelism_hint();
        source.setParallelism(dop);
      } else {
        common.set_parallelism_hint(1);
      }
    }

    /**
     * 1. Connect all spout streams with bolts streams 2. Then proceed with the bolts stream already
     * connected
     *
     * <p>Because we do not know the order in which an iterator steps over a set, we might process a
     * consumer before its producer ->thus, we might need to repeat multiple times
     */
    boolean makeProgress = true;
    while (bolts.size() > 0) {
      if (!makeProgress) {
        StringBuilder strBld = new StringBuilder();
        strBld.append("Unable to build Topology. Could not connect the following bolts:");
        for (String boltId : bolts.keySet()) {
          strBld.append("\n  ");
          strBld.append(boltId);
          strBld.append(": missing input streams [");
          for (Entry<GlobalStreamId, Grouping> streams : unprocessdInputsPerBolt.get(boltId)) {
            strBld.append("'");
            strBld.append(streams.getKey().get_streamId());
            strBld.append("' from '");
            strBld.append(streams.getKey().get_componentId());
            strBld.append("'; ");
          }
          strBld.append("]");
        }

        throw new RuntimeException(strBld.toString());
      }
      makeProgress = false;

      final Iterator<Entry<String, IRichBolt>> boltsIterator = bolts.entrySet().iterator();
      while (boltsIterator.hasNext()) {

        final Entry<String, IRichBolt> bolt = boltsIterator.next();
        final String boltId = bolt.getKey();
        final IRichBolt userBolt = copyObject(bolt.getValue());

        final ComponentCommon common = stormTopology.get_bolts().get(boltId).get_common();

        Set<Entry<GlobalStreamId, Grouping>> unprocessedBoltInputs =
            unprocessdInputsPerBolt.get(boltId);
        if (unprocessedBoltInputs == null) {
          unprocessedBoltInputs = new HashSet<>();
          unprocessedBoltInputs.addAll(common.get_inputs().entrySet());
          unprocessdInputsPerBolt.put(boltId, unprocessedBoltInputs);
        }

        // check if all inputs are available
        final int numberOfInputs = unprocessedBoltInputs.size();
        int inputsAvailable = 0;
        for (Entry<GlobalStreamId, Grouping> entry : unprocessedBoltInputs) {
          final String producerId = entry.getKey().get_componentId();
          final String streamId = entry.getKey().get_streamId();
          final HashMap<String, DataStream<Tuple>> streams = availableInputs.get(producerId);
          if (streams != null && streams.get(streamId) != null) {
            inputsAvailable++;
          }
        }

        if (inputsAvailable != numberOfInputs) {
          // traverse other bolts first until inputs are available
          continue;
        } else {
          makeProgress = true;
          boltsIterator.remove();
        }

        final Map<GlobalStreamId, DataStream<Tuple>> inputStreams = new HashMap<>(numberOfInputs);

        for (Entry<GlobalStreamId, Grouping> input : unprocessedBoltInputs) {
          final GlobalStreamId streamId = input.getKey();
          final Grouping grouping = input.getValue();

          final String producerId = streamId.get_componentId();

          final Map<String, DataStream<Tuple>> producer = availableInputs.get(producerId);

          inputStreams.put(streamId, processInput(boltId, userBolt, streamId, grouping, producer));
        }

        final SingleOutputStreamOperator<?, ?> outputStream =
            createOutput(boltId, userBolt, inputStreams);

        if (common.is_set_parallelism_hint()) {
          int dop = common.get_parallelism_hint();
          outputStream.setParallelism(dop);
        } else {
          common.set_parallelism_hint(1);
        }
      }
    }
  }
Example #9
0
  /**
   * Returns a representation of the non-system components of the topology graph Each Component
   * object in the returning map is populated with the list of its parents, children and execs
   * assigned to that component.
   *
   * @return a map of components
   */
  public Map<String, Component> getComponents() {
    Map<String, Component> all_comp = new HashMap<String, Component>();

    StormTopology storm_topo = this.topology;
    // spouts
    if (storm_topo.get_spouts() != null) {
      for (Map.Entry<String, SpoutSpec> spoutEntry : storm_topo.get_spouts().entrySet()) {
        if (!Utils.isSystemId(spoutEntry.getKey())) {
          Component newComp = null;
          if (all_comp.containsKey(spoutEntry.getKey())) {
            newComp = all_comp.get(spoutEntry.getKey());
            newComp.execs = componentToExecs(newComp.id);
          } else {
            newComp = new Component(spoutEntry.getKey());
            newComp.execs = componentToExecs(newComp.id);
            all_comp.put(spoutEntry.getKey(), newComp);
          }
          newComp.type = Component.ComponentType.SPOUT;

          for (Map.Entry<GlobalStreamId, Grouping> spoutInput :
              spoutEntry.getValue().get_common().get_inputs().entrySet()) {
            newComp.parents.add(spoutInput.getKey().get_componentId());
            if (!all_comp.containsKey(spoutInput.getKey().get_componentId())) {
              all_comp.put(
                  spoutInput.getKey().get_componentId(),
                  new Component(spoutInput.getKey().get_componentId()));
            }
            all_comp.get(spoutInput.getKey().get_componentId()).children.add(spoutEntry.getKey());
          }
        }
      }
    }
    // bolts
    if (storm_topo.get_bolts() != null) {
      for (Map.Entry<String, Bolt> boltEntry : storm_topo.get_bolts().entrySet()) {
        if (!Utils.isSystemId(boltEntry.getKey())) {
          Component newComp = null;
          if (all_comp.containsKey(boltEntry.getKey())) {
            newComp = all_comp.get(boltEntry.getKey());
            newComp.execs = componentToExecs(newComp.id);
          } else {
            newComp = new Component(boltEntry.getKey());
            newComp.execs = componentToExecs(newComp.id);
            all_comp.put(boltEntry.getKey(), newComp);
          }
          newComp.type = Component.ComponentType.BOLT;

          for (Map.Entry<GlobalStreamId, Grouping> boltInput :
              boltEntry.getValue().get_common().get_inputs().entrySet()) {
            newComp.parents.add(boltInput.getKey().get_componentId());
            if (!all_comp.containsKey(boltInput.getKey().get_componentId())) {
              all_comp.put(
                  boltInput.getKey().get_componentId(),
                  new Component(boltInput.getKey().get_componentId()));
            }
            all_comp.get(boltInput.getKey().get_componentId()).children.add(boltEntry.getKey());
          }
        }
      }
    }
    return all_comp;
  }