コード例 #1
0
  /** Get unstopped slots from alive task list 获得所有supervisor已经dead,但是还没有dead的任务 */
  public Set<Integer> getUnstoppedSlots(
      Set<Integer> aliveTasks, Map<String, SupervisorInfo> supInfos, Assignment existAssignment) {
    Set<Integer> ret = new HashSet<Integer>();

    Set<ResourceWorkerSlot> oldWorkers = existAssignment.getWorkers();

    Set<String> aliveSupervisors = supInfos.keySet();

    for (ResourceWorkerSlot worker : oldWorkers) {
      for (Integer taskId : worker.getTasks()) {
        if (aliveTasks.contains(taskId) == false) {
          // task is dead
          continue;
        }

        String oldTaskSupervisorId = worker.getNodeId();

        if (aliveSupervisors.contains(oldTaskSupervisorId) == false) {
          // supervisor is dead
          ret.add(taskId);
          continue;
        }
      }
    }

    return ret;
  }
コード例 #2
0
  /**
   * get all taskids which should be reassigned
   *
   * @param taskToWorkerSlot
   * @param newtaskToWorkerSlot
   * @return Set<Integer> taskid which should reassigned
   */
  public static Set<Integer> getChangeTaskIds(
      Set<ResourceWorkerSlot> oldWorkers, Set<ResourceWorkerSlot> workers) {

    Set<Integer> rtn = new HashSet<Integer>();
    for (ResourceWorkerSlot worker : workers) {
      if (!oldWorkers.contains(worker)) rtn.addAll(worker.getTasks());
    }
    return rtn;
  }
コード例 #3
0
 private Set<ResourceWorkerSlot> getUnstoppedWorkers(
     Set<Integer> aliveTasks, Assignment existAssignment) {
   Set<ResourceWorkerSlot> ret = new HashSet<ResourceWorkerSlot>();
   for (ResourceWorkerSlot worker : existAssignment.getWorkers()) {
     boolean alive = true;
     for (Integer task : worker.getTasks()) {
       if (!aliveTasks.contains(task)) {
         alive = false;
         break;
       }
     }
     if (alive) {
       ret.add(worker);
     }
   }
   return ret;
 }
コード例 #4
0
  /**
   * @param existingAssignment
   * @param taskWorkerSlot
   * @return
   * @throws Exception
   */
  public static Map<Integer, Integer> getTaskStartTimes(
      TopologyAssignContext context,
      NimbusData nimbusData,
      String topologyId,
      Assignment existingAssignment,
      Set<ResourceWorkerSlot> workers)
      throws Exception {

    Map<Integer, Integer> startTimes = new TreeMap<Integer, Integer>();

    if (context.getAssignType() == TopologyAssignContext.ASSIGN_TYPE_NEW) {
      int nowSecs = TimeUtils.current_time_secs();
      for (ResourceWorkerSlot worker : workers) {
        for (Integer changedTaskId : worker.getTasks()) {
          startTimes.put(changedTaskId, nowSecs);
        }
      }

      return startTimes;
    }

    Set<ResourceWorkerSlot> oldWorkers = new HashSet<ResourceWorkerSlot>();

    if (existingAssignment != null) {
      Map<Integer, Integer> taskStartTimeSecs = existingAssignment.getTaskStartTimeSecs();
      if (taskStartTimeSecs != null) {
        startTimes.putAll(taskStartTimeSecs);
      }

      if (existingAssignment.getWorkers() != null) {
        oldWorkers = existingAssignment.getWorkers();
      }
    }

    StormClusterState zkClusterState = nimbusData.getStormClusterState();
    Set<Integer> changeTaskIds = getChangeTaskIds(oldWorkers, workers);
    int nowSecs = TimeUtils.current_time_secs();
    for (Integer changedTaskId : changeTaskIds) {
      startTimes.put(changedTaskId, nowSecs);

      zkClusterState.remove_task_heartbeat(topologyId, changedTaskId);
    }

    LOG.info("Task assignment has been changed " + changeTaskIds);
    return startTimes;
  }
コード例 #5
0
 private static Set<ResourceWorkerSlot> mkLocalAssignment(TopologyAssignContext context) {
   Set<ResourceWorkerSlot> result = new HashSet<ResourceWorkerSlot>();
   Map<String, SupervisorInfo> cluster = context.getCluster();
   if (cluster.size() != 1) throw new RuntimeException();
   SupervisorInfo localSupervisor = null;
   String supervisorId = null;
   for (Entry<String, SupervisorInfo> entry : cluster.entrySet()) {
     supervisorId = entry.getKey();
     localSupervisor = entry.getValue();
   }
   int port = localSupervisor.getWorkerPorts().iterator().next();
   ResourceWorkerSlot worker = new ResourceWorkerSlot(supervisorId, port);
   worker.setTasks(new HashSet<Integer>(context.getAllTaskIds()));
   worker.setHostname(localSupervisor.getHostName());
   result.add(worker);
   return result;
 }
コード例 #6
0
  /**
   * Get free resources 获得所有的空闲资源
   *
   * @param supervisorInfos
   * @param stormClusterState
   * @throws Exception
   */
  public static void getFreeSlots(
      Map<String, SupervisorInfo> supervisorInfos, StormClusterState stormClusterState)
      throws Exception {

    Map<String, Assignment> assignments = Cluster.get_all_assignment(stormClusterState, null);

    for (Entry<String, Assignment> entry : assignments.entrySet()) {
      String topologyId = entry.getKey();
      Assignment assignment = entry.getValue();

      Set<ResourceWorkerSlot> workers = assignment.getWorkers();

      for (ResourceWorkerSlot worker : workers) {

        SupervisorInfo supervisorInfo = supervisorInfos.get(worker.getNodeId());
        if (supervisorInfo == null) {
          // the supervisor is dead
          continue;
        }
        supervisorInfo.getWorkerPorts().remove(worker.getPort());
      }
    }
  }
コード例 #7
0
  public static Map<String, String> getTopologyNodeHost(
      Map<String, SupervisorInfo> supervisorMap,
      Assignment existingAssignment,
      Set<ResourceWorkerSlot> workers) {

    // the following is that remove unused node from allNodeHost
    Set<String> usedNodes = new HashSet<String>();
    for (ResourceWorkerSlot worker : workers) {

      usedNodes.add(worker.getNodeId());
    }

    // map<supervisorId, hostname>
    Map<String, String> allNodeHost = new HashMap<String, String>();

    if (existingAssignment != null) {
      allNodeHost.putAll(existingAssignment.getNodeHost());
    }

    // get alive supervisorMap Map<supervisorId, hostname>
    Map<String, String> nodeHost = SupervisorInfo.getNodeHost(supervisorMap);
    if (nodeHost != null) {
      allNodeHost.putAll(nodeHost);
    }

    Map<String, String> ret = new HashMap<String, String>();

    for (String supervisorId : usedNodes) {
      if (allNodeHost.containsKey(supervisorId)) {
        ret.put(supervisorId, allNodeHost.get(supervisorId));
      } else {
        LOG.warn("Node " + supervisorId + " doesn't in the supervisor list");
      }
    }

    return ret;
  }