Example #1
0
  /**
   * cleanup the topologies which are not in ZK /topology, but in other place
   *
   * @param nimbusData
   * @param active_topologys
   * @throws Exception
   */
  public void cleanupDisappearedTopology() throws Exception {
    StormClusterState clusterState = nimbusData.getStormClusterState();

    List<String> active_topologys = clusterState.active_storms();
    if (active_topologys == null) {
      return;
    }

    Set<String> cleanupIds = get_cleanup_ids(clusterState, active_topologys);

    for (String topologyId : cleanupIds) {

      LOG.info("Cleaning up " + topologyId);

      clusterState.try_remove_storm(topologyId);
      //
      nimbusData.getTaskHeartbeatsCache().remove(topologyId);

      // get /nimbus/stormdist/topologyId
      String master_stormdist_root =
          StormConfig.masterStormdistRoot(nimbusData.getConf(), topologyId);
      try {
        // delete topologyId local dir
        PathUtils.rmr(master_stormdist_root);
      } catch (IOException e) {
        LOG.warn("Failed to delete " + master_stormdist_root + ",", e);
      }
    }
  }
Example #2
0
  /**
   * make assignments for a topology The nimbus core function, this function has been totally
   * rewrite
   *
   * @param nimbusData NimbusData
   * @param topologyId String
   * @param isScratch Boolean: isScratch is false unless rebalancing the topology
   * @throws Exception
   */
  public Assignment mkAssignment(TopologyAssignEvent event) throws Exception {
    String topologyId = event.getTopologyId();

    LOG.info("Determining assignment for " + topologyId);

    TopologyAssignContext context = prepareTopologyAssign(event);

    Set<ResourceWorkerSlot> assignments = null;

    if (!StormConfig.local_mode(nimbusData.getConf())) {

      IToplogyScheduler scheduler = schedulers.get(DEFAULT_SCHEDULER_NAME);

      assignments = scheduler.assignTasks(context);

    } else {
      assignments = mkLocalAssignment(context);
    }
    Assignment assignment = null;

    Map<String, String> nodeHost =
        getTopologyNodeHost(context.getCluster(), context.getOldAssignment(), assignments);

    Map<Integer, Integer> startTimes =
        getTaskStartTimes(context, nimbusData, topologyId, context.getOldAssignment(), assignments);

    String codeDir = StormConfig.masterStormdistRoot(nimbusData.getConf(), topologyId);

    assignment = new Assignment(codeDir, assignments, nodeHost, startTimes);

    StormClusterState stormClusterState = nimbusData.getStormClusterState();

    stormClusterState.set_assignment(topologyId, assignment);

    // update task heartbeat's start time
    NimbusUtils.updateTaskHbStartTime(nimbusData, assignment, topologyId);

    // Update metrics information in ZK when rebalance or reassignment
    // Only update metrics monitor status when creating topology
    if (context.getAssignType() == TopologyAssignContext.ASSIGN_TYPE_REBALANCE
        || context.getAssignType() == TopologyAssignContext.ASSIGN_TYPE_MONITOR)
      NimbusUtils.updateMetricsInfo(nimbusData, topologyId, assignment);
    else metricsMonitor(event);

    LOG.info("Successfully make assignment for topology id " + topologyId + ": " + assignment);

    return assignment;
  }
Example #3
0
  /**
   * get topology ids which need to be cleanup
   *
   * @param clusterState
   * @return
   * @throws Exception
   */
  private Set<String> get_cleanup_ids(StormClusterState clusterState, List<String> active_topologys)
      throws Exception {

    List<String> task_ids = clusterState.task_storms();
    List<String> heartbeat_ids = clusterState.heartbeat_storms();
    List<String> error_ids = clusterState.task_error_storms();
    List<String> assignment_ids = clusterState.assignments(null);
    List<String> monitor_ids = clusterState.monitors();

    String master_stormdist_root = StormConfig.masterStormdistRoot(nimbusData.getConf());
    // listdir /local-dir/nimbus/stormdist
    List<String> code_ids = PathUtils.read_dir_contents(master_stormdist_root);

    // Set<String> assigned_ids =
    // JStormUtils.listToSet(clusterState.active_storms());
    Set<String> to_cleanup_ids = new HashSet<String>();

    if (task_ids != null) {
      to_cleanup_ids.addAll(task_ids);
    }

    if (heartbeat_ids != null) {
      to_cleanup_ids.addAll(heartbeat_ids);
    }

    if (error_ids != null) {
      to_cleanup_ids.addAll(error_ids);
    }

    if (assignment_ids != null) {
      to_cleanup_ids.addAll(assignment_ids);
    }

    if (monitor_ids != null) {
      to_cleanup_ids.addAll(monitor_ids);
    }

    if (code_ids != null) {
      to_cleanup_ids.addAll(code_ids);
    }

    if (active_topologys != null) {
      to_cleanup_ids.removeAll(active_topologys);
    }
    return to_cleanup_ids;
  }
Example #4
0
  /**
   * create local topology files /local-dir/nimbus/topologyId/stormjar.jar
   * /local-dir/nimbus/topologyId/stormcode.ser /local-dir/nimbus/topologyId/stormconf.ser
   *
   * @param conf
   * @param topologyId
   * @param tmpJarLocation
   * @param stormConf
   * @param topology
   * @throws IOException
   */
  private void setupStormCode(
      Map<Object, Object> conf,
      String topologyId,
      String tmpJarLocation,
      Map<Object, Object> stormConf,
      StormTopology topology)
      throws IOException {
    // local-dir/nimbus/stormdist/topologyId
    String stormroot = StormConfig.masterStormdistRoot(conf, topologyId);

    FileUtils.forceMkdir(new File(stormroot));
    FileUtils.cleanDirectory(new File(stormroot));

    // copy jar to /local-dir/nimbus/topologyId/stormjar.jar
    setupJar(conf, tmpJarLocation, stormroot);

    // serialize to file /local-dir/nimbus/topologyId/stormcode.ser
    FileUtils.writeByteArrayToFile(
        new File(StormConfig.stormcode_path(stormroot)), Utils.serialize(topology));

    // serialize to file /local-dir/nimbus/topologyId/stormconf.ser
    FileUtils.writeByteArrayToFile(
        new File(StormConfig.sotrmconf_path(stormroot)), Utils.serialize(stormConf));
  }