private TaskCluster getTaskCluster(TaskId tid) {
   JobRun run = scheduler.getJobRun();
   ActivityCluster ac = run.getActivityClusterGraph().getActivityMap().get(tid.getActivityId());
   ActivityClusterPlan acp = run.getActivityClusterPlanMap().get(ac.getId());
   Task[] tasks = acp.getActivityPlanMap().get(tid.getActivityId()).getTasks();
   Task task = tasks[tid.getPartition()];
   assert task.getTaskId().equals(tid);
   return task.getTaskCluster();
 }
 private TaskCluster[] buildConnectorPolicyUnawareTaskClusters(
     ActivityCluster ac, Map<ActivityId, ActivityPlan> activityPlanMap) {
   List<Task> taskStates = new ArrayList<Task>();
   for (ActivityId anId : ac.getActivityMap().keySet()) {
     ActivityPlan ap = activityPlanMap.get(anId);
     Task[] tasks = ap.getTasks();
     for (Task t : tasks) {
       taskStates.add(t);
     }
   }
   TaskCluster tc =
       new TaskCluster(
           new TaskClusterId(ac.getId(), 0), ac, taskStates.toArray(new Task[taskStates.size()]));
   for (Task t : tc.getTasks()) {
     t.setTaskCluster(tc);
   }
   return new TaskCluster[] {tc};
 }
  private TaskCluster[] computeTaskClusters(
      ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPlan> activityPlanMap) {
    Set<ActivityId> activities = ac.getActivityMap().keySet();
    Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity =
        computeTaskConnectivity(jobRun, activityPlanMap, activities);

    TaskCluster[] taskClusters =
        ac.getActivityClusterGraph().isUseConnectorPolicyForScheduling()
            ? buildConnectorPolicyAwareTaskClusters(ac, activityPlanMap, taskConnectivity)
            : buildConnectorPolicyUnawareTaskClusters(ac, activityPlanMap);

    for (TaskCluster tc : taskClusters) {
      Set<TaskCluster> tcDependencyTaskClusters = tc.getDependencyTaskClusters();
      for (Task ts : tc.getTasks()) {
        TaskId tid = ts.getTaskId();
        List<Pair<TaskId, ConnectorDescriptorId>> cInfoList = taskConnectivity.get(tid);
        if (cInfoList != null) {
          for (Pair<TaskId, ConnectorDescriptorId> p : cInfoList) {
            Task targetTS =
                activityPlanMap.get(p.getLeft().getActivityId())
                    .getTasks()[p.getLeft().getPartition()];
            TaskCluster targetTC = targetTS.getTaskCluster();
            if (targetTC != tc) {
              ConnectorDescriptorId cdId = p.getRight();
              PartitionId pid =
                  new PartitionId(
                      jobRun.getJobId(), cdId, tid.getPartition(), p.getLeft().getPartition());
              tc.getProducedPartitions().add(pid);
              targetTC.getRequiredPartitions().add(pid);
              partitionProducingTaskClusterMap.put(pid, tc);
            }
          }
        }

        for (TaskId dTid : ts.getDependencies()) {
          TaskCluster dTC = getTaskCluster(dTid);
          dTC.getDependentTaskClusters().add(tc);
          tcDependencyTaskClusters.add(dTC);
        }
      }
    }
    return taskClusters;
  }
 private Map<ActivityId, ActivityPlan> buildActivityPlanMap(
     ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPartitionDetails> pcMap) {
   Map<ActivityId, ActivityPlan> activityPlanMap = new HashMap<ActivityId, ActivityPlan>();
   Set<ActivityId> depAnIds = new HashSet<ActivityId>();
   for (ActivityId anId : ac.getActivityMap().keySet()) {
     depAnIds.clear();
     getDependencyActivityIds(depAnIds, anId, ac);
     ActivityPartitionDetails apd = pcMap.get(anId);
     Task[] tasks = new Task[apd.getPartitionCount()];
     ActivityPlan activityPlan = new ActivityPlan(apd);
     for (int i = 0; i < tasks.length; ++i) {
       TaskId tid = new TaskId(anId, i);
       tasks[i] = new Task(tid, activityPlan);
       for (ActivityId danId : depAnIds) {
         ActivityCluster dAC = ac.getActivityClusterGraph().getActivityMap().get(danId);
         ActivityClusterPlan dACP = jobRun.getActivityClusterPlanMap().get(dAC.getId());
         assert dACP != null
             : "IllegalStateEncountered: Dependent AC is being planned without a plan for dependency AC: Encountered no plan for ActivityID "
                 + danId;
         Task[] dATasks = dACP.getActivityPlanMap().get(danId).getTasks();
         assert dATasks != null
             : "IllegalStateEncountered: Dependent AC is being planned without a plan for dependency AC: Encountered no plan for ActivityID "
                 + danId;
         assert dATasks.length == tasks.length
             : "Dependency activity partitioned differently from dependent: "
                 + dATasks.length
                 + " != "
                 + tasks.length;
         Task dTask = dATasks[i];
         TaskId dTaskId = dTask.getTaskId();
         tasks[i].getDependencies().add(dTaskId);
         dTask.getDependents().add(tid);
       }
     }
     activityPlan.setTasks(tasks);
     activityPlanMap.put(anId, activityPlan);
   }
   return activityPlanMap;
 }
  private TaskCluster[] buildConnectorPolicyAwareTaskClusters(
      ActivityCluster ac,
      Map<ActivityId, ActivityPlan> activityPlanMap,
      Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity) {
    Map<TaskId, Set<TaskId>> taskClusterMap = new HashMap<TaskId, Set<TaskId>>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
      ActivityPlan ap = activityPlanMap.get(anId);
      Task[] tasks = ap.getTasks();
      for (Task t : tasks) {
        Set<TaskId> cluster = new HashSet<TaskId>();
        TaskId tid = t.getTaskId();
        cluster.add(tid);
        taskClusterMap.put(tid, cluster);
      }
    }

    JobRun jobRun = scheduler.getJobRun();
    Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
    for (Map.Entry<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> e :
        taskConnectivity.entrySet()) {
      Set<TaskId> cluster = taskClusterMap.get(e.getKey());
      for (Pair<TaskId, ConnectorDescriptorId> p : e.getValue()) {
        IConnectorPolicy cPolicy = connectorPolicies.get(p.getRight());
        if (cPolicy.requiresProducerConsumerCoscheduling()) {
          cluster.add(p.getLeft());
        }
      }
    }

    /*
     * taskClusterMap contains for every TID x, x -> { coscheduled consumer TIDs U x }
     * We compute the transitive closure of this relation to find the largest set of
     * tasks that need to be co-scheduled
     */
    int counter = 0;
    TaskId[] ordinalList = new TaskId[taskClusterMap.size()];
    Map<TaskId, Integer> ordinalMap = new HashMap<TaskId, Integer>();
    for (TaskId tid : taskClusterMap.keySet()) {
      ordinalList[counter] = tid;
      ordinalMap.put(tid, counter);
      ++counter;
    }

    int n = ordinalList.length;
    BitSet[] paths = new BitSet[n];
    for (Map.Entry<TaskId, Set<TaskId>> e : taskClusterMap.entrySet()) {
      int i = ordinalMap.get(e.getKey());
      BitSet bsi = paths[i];
      if (bsi == null) {
        bsi = new BitSet(n);
        paths[i] = bsi;
      }
      for (TaskId ttid : e.getValue()) {
        int j = ordinalMap.get(ttid);
        paths[i].set(j);
        BitSet bsj = paths[j];
        if (bsj == null) {
          bsj = new BitSet(n);
          paths[j] = bsj;
        }
        bsj.set(i);
      }
    }
    for (int k = 0; k < n; ++k) {
      for (int i = paths[k].nextSetBit(0); i >= 0; i = paths[k].nextSetBit(i + 1)) {
        for (int j = paths[i].nextClearBit(0); j < n && j >= 0; j = paths[i].nextClearBit(j + 1)) {
          paths[i].set(j, paths[k].get(j));
          paths[j].set(i, paths[i].get(j));
        }
      }
    }
    BitSet pending = new BitSet(n);
    pending.set(0, n);
    List<List<TaskId>> clusters = new ArrayList<List<TaskId>>();
    for (int i = pending.nextSetBit(0); i >= 0; i = pending.nextSetBit(i)) {
      List<TaskId> cluster = new ArrayList<TaskId>();
      for (int j = paths[i].nextSetBit(0); j >= 0; j = paths[i].nextSetBit(j + 1)) {
        cluster.add(ordinalList[j]);
        pending.clear(j);
      }
      clusters.add(cluster);
    }

    List<TaskCluster> tcSet = new ArrayList<TaskCluster>();
    counter = 0;
    for (List<TaskId> cluster : clusters) {
      List<Task> taskStates = new ArrayList<Task>();
      for (TaskId tid : cluster) {
        taskStates.add(activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()]);
      }
      TaskCluster tc =
          new TaskCluster(
              new TaskClusterId(ac.getId(), counter++),
              ac,
              taskStates.toArray(new Task[taskStates.size()]));
      tcSet.add(tc);
      for (TaskId tid : cluster) {
        activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()].setTaskCluster(tc);
      }
    }
    TaskCluster[] taskClusters = tcSet.toArray(new TaskCluster[tcSet.size()]);
    return taskClusters;
  }