private IConnectorPolicy assignConnectorPolicy(
ActivityCluster ac, IConnectorDescriptor c, int nProducers, int nConsumers, int[] fanouts) {
IConnectorPolicyAssignmentPolicy cpap = ac.getConnectorPolicyAssignmentPolicy();
if (cpap != null) {
return cpap.getConnectorPolicyAssignment(c, nProducers, nConsumers, fanouts);
}
cpap = ac.getActivityClusterGraph().getConnectorPolicyAssignmentPolicy();
if (cpap != null) {
return cpap.getConnectorPolicyAssignment(c, nProducers, nConsumers, fanouts);
}
return new PipeliningConnectorPolicy();
}
private TaskCluster[] computeTaskClusters(
ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPlan> activityPlanMap) {
Set<ActivityId> activities = ac.getActivityMap().keySet();
Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity =
computeTaskConnectivity(jobRun, activityPlanMap, activities);
TaskCluster[] taskClusters =
ac.getActivityClusterGraph().isUseConnectorPolicyForScheduling()
? buildConnectorPolicyAwareTaskClusters(ac, activityPlanMap, taskConnectivity)
: buildConnectorPolicyUnawareTaskClusters(ac, activityPlanMap);
for (TaskCluster tc : taskClusters) {
Set<TaskCluster> tcDependencyTaskClusters = tc.getDependencyTaskClusters();
for (Task ts : tc.getTasks()) {
TaskId tid = ts.getTaskId();
List<Pair<TaskId, ConnectorDescriptorId>> cInfoList = taskConnectivity.get(tid);
if (cInfoList != null) {
for (Pair<TaskId, ConnectorDescriptorId> p : cInfoList) {
Task targetTS =
activityPlanMap.get(p.getLeft().getActivityId())
.getTasks()[p.getLeft().getPartition()];
TaskCluster targetTC = targetTS.getTaskCluster();
if (targetTC != tc) {
ConnectorDescriptorId cdId = p.getRight();
PartitionId pid =
new PartitionId(
jobRun.getJobId(), cdId, tid.getPartition(), p.getLeft().getPartition());
tc.getProducedPartitions().add(pid);
targetTC.getRequiredPartitions().add(pid);
partitionProducingTaskClusterMap.put(pid, tc);
}
}
}
for (TaskId dTid : ts.getDependencies()) {
TaskCluster dTC = getTaskCluster(dTid);
dTC.getDependentTaskClusters().add(tc);
tcDependencyTaskClusters.add(dTC);
}
}
}
return taskClusters;
}
private Map<ActivityId, ActivityPlan> buildActivityPlanMap(
ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPartitionDetails> pcMap) {
Map<ActivityId, ActivityPlan> activityPlanMap = new HashMap<ActivityId, ActivityPlan>();
Set<ActivityId> depAnIds = new HashSet<ActivityId>();
for (ActivityId anId : ac.getActivityMap().keySet()) {
depAnIds.clear();
getDependencyActivityIds(depAnIds, anId, ac);
ActivityPartitionDetails apd = pcMap.get(anId);
Task[] tasks = new Task[apd.getPartitionCount()];
ActivityPlan activityPlan = new ActivityPlan(apd);
for (int i = 0; i < tasks.length; ++i) {
TaskId tid = new TaskId(anId, i);
tasks[i] = new Task(tid, activityPlan);
for (ActivityId danId : depAnIds) {
ActivityCluster dAC = ac.getActivityClusterGraph().getActivityMap().get(danId);
ActivityClusterPlan dACP = jobRun.getActivityClusterPlanMap().get(dAC.getId());
assert dACP != null
: "IllegalStateEncountered: Dependent AC is being planned without a plan for dependency AC: Encountered no plan for ActivityID "
+ danId;
Task[] dATasks = dACP.getActivityPlanMap().get(danId).getTasks();
assert dATasks != null
: "IllegalStateEncountered: Dependent AC is being planned without a plan for dependency AC: Encountered no plan for ActivityID "
+ danId;
assert dATasks.length == tasks.length
: "Dependency activity partitioned differently from dependent: "
+ dATasks.length
+ " != "
+ tasks.length;
Task dTask = dATasks[i];
TaskId dTaskId = dTask.getTaskId();
tasks[i].getDependencies().add(dTaskId);
dTask.getDependents().add(tid);
}
}
activityPlan.setTasks(tasks);
activityPlanMap.put(anId, activityPlan);
}
return activityPlanMap;
}