private void assignConnectorPolicy(ActivityCluster ac, Map<ActivityId, ActivityPlan> taskMap) {
Map<ConnectorDescriptorId, IConnectorPolicy> cPolicyMap =
new HashMap<ConnectorDescriptorId, IConnectorPolicy>();
Set<ActivityId> activities = ac.getActivityMap().keySet();
BitSet targetBitmap = new BitSet();
for (ActivityId a1 : activities) {
Task[] ac1TaskStates = taskMap.get(a1).getTasks();
int nProducers = ac1TaskStates.length;
List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(a1);
if (outputConns != null) {
for (IConnectorDescriptor c : outputConns) {
ConnectorDescriptorId cdId = c.getConnectorId();
ActivityId a2 = ac.getConsumerActivity(cdId);
Task[] ac2TaskStates = taskMap.get(a2).getTasks();
int nConsumers = ac2TaskStates.length;
int[] fanouts = new int[nProducers];
if (c.allProducersToAllConsumers()) {
for (int i = 0; i < nProducers; ++i) {
fanouts[i] = nConsumers;
}
} else {
for (int i = 0; i < nProducers; ++i) {
c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
fanouts[i] = targetBitmap.cardinality();
}
}
IConnectorPolicy cp = assignConnectorPolicy(ac, c, nProducers, nConsumers, fanouts);
cPolicyMap.put(cdId, cp);
}
}
}
scheduler.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
}
private Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> computeTaskConnectivity(
JobRun jobRun, Map<ActivityId, ActivityPlan> activityPlanMap, Set<ActivityId> activities) {
Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity =
new HashMap<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>>();
ActivityClusterGraph acg = jobRun.getActivityClusterGraph();
BitSet targetBitmap = new BitSet();
for (ActivityId ac1 : activities) {
ActivityCluster ac = acg.getActivityMap().get(ac1);
Task[] ac1TaskStates = activityPlanMap.get(ac1).getTasks();
int nProducers = ac1TaskStates.length;
List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(ac1);
if (outputConns != null) {
for (IConnectorDescriptor c : outputConns) {
ConnectorDescriptorId cdId = c.getConnectorId();
ActivityId ac2 = ac.getConsumerActivity(cdId);
Task[] ac2TaskStates = activityPlanMap.get(ac2).getTasks();
int nConsumers = ac2TaskStates.length;
if (c.allProducersToAllConsumers()) {
List<Pair<TaskId, ConnectorDescriptorId>> cInfoList =
new ArrayList<Pair<TaskId, ConnectorDescriptorId>>();
for (int j = 0; j < nConsumers; j++) {
TaskId targetTID = ac2TaskStates[j].getTaskId();
cInfoList.add(Pair.<TaskId, ConnectorDescriptorId>of(targetTID, cdId));
}
for (int i = 0; i < nProducers; ++i) {
taskConnectivity.put(ac1TaskStates[i].getTaskId(), cInfoList);
}
} else {
for (int i = 0; i < nProducers; ++i) {
c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
List<Pair<TaskId, ConnectorDescriptorId>> cInfoList =
taskConnectivity.get(ac1TaskStates[i].getTaskId());
if (cInfoList == null) {
cInfoList = new ArrayList<Pair<TaskId, ConnectorDescriptorId>>();
taskConnectivity.put(ac1TaskStates[i].getTaskId(), cInfoList);
}
for (int j = targetBitmap.nextSetBit(0); j >= 0; j = targetBitmap.nextSetBit(j + 1)) {
TaskId targetTID = ac2TaskStates[j].getTaskId();
cInfoList.add(Pair.<TaskId, ConnectorDescriptorId>of(targetTID, cdId));
}
}
}
}
}
}
return taskConnectivity;
}
private Map<ActivityId, ActivityPartitionDetails> computePartitionCounts(ActivityCluster ac)
throws HyracksException {
PartitionConstraintSolver solver = scheduler.getSolver();
Set<LValueConstraintExpression> lValues = new HashSet<LValueConstraintExpression>();
for (ActivityId anId : ac.getActivityMap().keySet()) {
lValues.add(new PartitionCountExpression(anId.getOperatorDescriptorId()));
}
solver.solve(lValues);
Map<OperatorDescriptorId, Integer> nPartMap = new HashMap<OperatorDescriptorId, Integer>();
for (LValueConstraintExpression lv : lValues) {
Object value = solver.getValue(lv);
if (value == null) {
throw new HyracksException("No value found for " + lv);
}
if (!(value instanceof Number)) {
throw new HyracksException(
"Unexpected type of value bound to "
+ lv
+ ": "
+ value.getClass()
+ "("
+ value
+ ")");
}
int nParts = ((Number) value).intValue();
if (nParts <= 0) {
throw new HyracksException("Unsatisfiable number of partitions for " + lv + ": " + nParts);
}
nPartMap.put(
((PartitionCountExpression) lv).getOperatorDescriptorId(), Integer.valueOf(nParts));
}
Map<ActivityId, ActivityPartitionDetails> activityPartsMap =
new HashMap<ActivityId, ActivityPartitionDetails>();
for (ActivityId anId : ac.getActivityMap().keySet()) {
int nParts = nPartMap.get(anId.getOperatorDescriptorId());
int[] nInputPartitions = null;
List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(anId);
if (inputs != null) {
nInputPartitions = new int[inputs.size()];
for (int i = 0; i < nInputPartitions.length; ++i) {
ConnectorDescriptorId cdId = inputs.get(i).getConnectorId();
ActivityId aid = ac.getProducerActivity(cdId);
Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId());
nInputPartitions[i] = nPartInt;
}
}
int[] nOutputPartitions = null;
List<IConnectorDescriptor> outputs = ac.getActivityOutputMap().get(anId);
if (outputs != null) {
nOutputPartitions = new int[outputs.size()];
for (int i = 0; i < nOutputPartitions.length; ++i) {
ConnectorDescriptorId cdId = outputs.get(i).getConnectorId();
ActivityId aid = ac.getConsumerActivity(cdId);
Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId());
nOutputPartitions[i] = nPartInt;
}
}
ActivityPartitionDetails apd =
new ActivityPartitionDetails(nParts, nInputPartitions, nOutputPartitions);
activityPartsMap.put(anId, apd);
}
return activityPartsMap;
}