コード例 #1
0
 private TaskCluster getTaskCluster(TaskId tid) {
   JobRun run = scheduler.getJobRun();
   ActivityCluster ac = run.getActivityClusterGraph().getActivityMap().get(tid.getActivityId());
   ActivityClusterPlan acp = run.getActivityClusterPlanMap().get(ac.getId());
   Task[] tasks = acp.getActivityPlanMap().get(tid.getActivityId()).getTasks();
   Task task = tasks[tid.getPartition()];
   assert task.getTaskId().equals(tid);
   return task.getTaskCluster();
 }
コード例 #2
0
  private TaskCluster[] computeTaskClusters(
      ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPlan> activityPlanMap) {
    Set<ActivityId> activities = ac.getActivityMap().keySet();
    Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity =
        computeTaskConnectivity(jobRun, activityPlanMap, activities);

    TaskCluster[] taskClusters =
        ac.getActivityClusterGraph().isUseConnectorPolicyForScheduling()
            ? buildConnectorPolicyAwareTaskClusters(ac, activityPlanMap, taskConnectivity)
            : buildConnectorPolicyUnawareTaskClusters(ac, activityPlanMap);

    for (TaskCluster tc : taskClusters) {
      Set<TaskCluster> tcDependencyTaskClusters = tc.getDependencyTaskClusters();
      for (Task ts : tc.getTasks()) {
        TaskId tid = ts.getTaskId();
        List<Pair<TaskId, ConnectorDescriptorId>> cInfoList = taskConnectivity.get(tid);
        if (cInfoList != null) {
          for (Pair<TaskId, ConnectorDescriptorId> p : cInfoList) {
            Task targetTS =
                activityPlanMap.get(p.getLeft().getActivityId())
                    .getTasks()[p.getLeft().getPartition()];
            TaskCluster targetTC = targetTS.getTaskCluster();
            if (targetTC != tc) {
              ConnectorDescriptorId cdId = p.getRight();
              PartitionId pid =
                  new PartitionId(
                      jobRun.getJobId(), cdId, tid.getPartition(), p.getLeft().getPartition());
              tc.getProducedPartitions().add(pid);
              targetTC.getRequiredPartitions().add(pid);
              partitionProducingTaskClusterMap.put(pid, tc);
            }
          }
        }

        for (TaskId dTid : ts.getDependencies()) {
          TaskCluster dTC = getTaskCluster(dTid);
          dTC.getDependentTaskClusters().add(tc);
          tcDependencyTaskClusters.add(dTC);
        }
      }
    }
    return taskClusters;
  }
コード例 #3
0
  private TaskCluster[] buildConnectorPolicyAwareTaskClusters(
      ActivityCluster ac,
      Map<ActivityId, ActivityPlan> activityPlanMap,
      Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity) {
    Map<TaskId, Set<TaskId>> taskClusterMap = new HashMap<TaskId, Set<TaskId>>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
      ActivityPlan ap = activityPlanMap.get(anId);
      Task[] tasks = ap.getTasks();
      for (Task t : tasks) {
        Set<TaskId> cluster = new HashSet<TaskId>();
        TaskId tid = t.getTaskId();
        cluster.add(tid);
        taskClusterMap.put(tid, cluster);
      }
    }

    JobRun jobRun = scheduler.getJobRun();
    Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
    for (Map.Entry<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> e :
        taskConnectivity.entrySet()) {
      Set<TaskId> cluster = taskClusterMap.get(e.getKey());
      for (Pair<TaskId, ConnectorDescriptorId> p : e.getValue()) {
        IConnectorPolicy cPolicy = connectorPolicies.get(p.getRight());
        if (cPolicy.requiresProducerConsumerCoscheduling()) {
          cluster.add(p.getLeft());
        }
      }
    }

    /*
     * taskClusterMap contains for every TID x, x -> { coscheduled consumer TIDs U x }
     * We compute the transitive closure of this relation to find the largest set of
     * tasks that need to be co-scheduled
     */
    int counter = 0;
    TaskId[] ordinalList = new TaskId[taskClusterMap.size()];
    Map<TaskId, Integer> ordinalMap = new HashMap<TaskId, Integer>();
    for (TaskId tid : taskClusterMap.keySet()) {
      ordinalList[counter] = tid;
      ordinalMap.put(tid, counter);
      ++counter;
    }

    int n = ordinalList.length;
    BitSet[] paths = new BitSet[n];
    for (Map.Entry<TaskId, Set<TaskId>> e : taskClusterMap.entrySet()) {
      int i = ordinalMap.get(e.getKey());
      BitSet bsi = paths[i];
      if (bsi == null) {
        bsi = new BitSet(n);
        paths[i] = bsi;
      }
      for (TaskId ttid : e.getValue()) {
        int j = ordinalMap.get(ttid);
        paths[i].set(j);
        BitSet bsj = paths[j];
        if (bsj == null) {
          bsj = new BitSet(n);
          paths[j] = bsj;
        }
        bsj.set(i);
      }
    }
    for (int k = 0; k < n; ++k) {
      for (int i = paths[k].nextSetBit(0); i >= 0; i = paths[k].nextSetBit(i + 1)) {
        for (int j = paths[i].nextClearBit(0); j < n && j >= 0; j = paths[i].nextClearBit(j + 1)) {
          paths[i].set(j, paths[k].get(j));
          paths[j].set(i, paths[i].get(j));
        }
      }
    }
    BitSet pending = new BitSet(n);
    pending.set(0, n);
    List<List<TaskId>> clusters = new ArrayList<List<TaskId>>();
    for (int i = pending.nextSetBit(0); i >= 0; i = pending.nextSetBit(i)) {
      List<TaskId> cluster = new ArrayList<TaskId>();
      for (int j = paths[i].nextSetBit(0); j >= 0; j = paths[i].nextSetBit(j + 1)) {
        cluster.add(ordinalList[j]);
        pending.clear(j);
      }
      clusters.add(cluster);
    }

    List<TaskCluster> tcSet = new ArrayList<TaskCluster>();
    counter = 0;
    for (List<TaskId> cluster : clusters) {
      List<Task> taskStates = new ArrayList<Task>();
      for (TaskId tid : cluster) {
        taskStates.add(activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()]);
      }
      TaskCluster tc =
          new TaskCluster(
              new TaskClusterId(ac.getId(), counter++),
              ac,
              taskStates.toArray(new Task[taskStates.size()]));
      tcSet.add(tc);
      for (TaskId tid : cluster) {
        activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()].setTaskCluster(tc);
      }
    }
    TaskCluster[] taskClusters = tcSet.toArray(new TaskCluster[tcSet.size()]);
    return taskClusters;
  }