private void assignConnectorPolicy(ActivityCluster ac, Map<ActivityId, ActivityPlan> taskMap) {
Map<ConnectorDescriptorId, IConnectorPolicy> cPolicyMap =
new HashMap<ConnectorDescriptorId, IConnectorPolicy>();
Set<ActivityId> activities = ac.getActivityMap().keySet();
BitSet targetBitmap = new BitSet();
for (ActivityId a1 : activities) {
Task[] ac1TaskStates = taskMap.get(a1).getTasks();
int nProducers = ac1TaskStates.length;
List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(a1);
if (outputConns != null) {
for (IConnectorDescriptor c : outputConns) {
ConnectorDescriptorId cdId = c.getConnectorId();
ActivityId a2 = ac.getConsumerActivity(cdId);
Task[] ac2TaskStates = taskMap.get(a2).getTasks();
int nConsumers = ac2TaskStates.length;
int[] fanouts = new int[nProducers];
if (c.allProducersToAllConsumers()) {
for (int i = 0; i < nProducers; ++i) {
fanouts[i] = nConsumers;
}
} else {
for (int i = 0; i < nProducers; ++i) {
c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
fanouts[i] = targetBitmap.cardinality();
}
}
IConnectorPolicy cp = assignConnectorPolicy(ac, c, nProducers, nConsumers, fanouts);
cPolicyMap.put(cdId, cp);
}
}
}
scheduler.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
}
private Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> computeTaskConnectivity(
JobRun jobRun, Map<ActivityId, ActivityPlan> activityPlanMap, Set<ActivityId> activities) {
Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity =
new HashMap<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>>();
ActivityClusterGraph acg = jobRun.getActivityClusterGraph();
BitSet targetBitmap = new BitSet();
for (ActivityId ac1 : activities) {
ActivityCluster ac = acg.getActivityMap().get(ac1);
Task[] ac1TaskStates = activityPlanMap.get(ac1).getTasks();
int nProducers = ac1TaskStates.length;
List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(ac1);
if (outputConns != null) {
for (IConnectorDescriptor c : outputConns) {
ConnectorDescriptorId cdId = c.getConnectorId();
ActivityId ac2 = ac.getConsumerActivity(cdId);
Task[] ac2TaskStates = activityPlanMap.get(ac2).getTasks();
int nConsumers = ac2TaskStates.length;
if (c.allProducersToAllConsumers()) {
List<Pair<TaskId, ConnectorDescriptorId>> cInfoList =
new ArrayList<Pair<TaskId, ConnectorDescriptorId>>();
for (int j = 0; j < nConsumers; j++) {
TaskId targetTID = ac2TaskStates[j].getTaskId();
cInfoList.add(Pair.<TaskId, ConnectorDescriptorId>of(targetTID, cdId));
}
for (int i = 0; i < nProducers; ++i) {
taskConnectivity.put(ac1TaskStates[i].getTaskId(), cInfoList);
}
} else {
for (int i = 0; i < nProducers; ++i) {
c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
List<Pair<TaskId, ConnectorDescriptorId>> cInfoList =
taskConnectivity.get(ac1TaskStates[i].getTaskId());
if (cInfoList == null) {
cInfoList = new ArrayList<Pair<TaskId, ConnectorDescriptorId>>();
taskConnectivity.put(ac1TaskStates[i].getTaskId(), cInfoList);
}
for (int j = targetBitmap.nextSetBit(0); j >= 0; j = targetBitmap.nextSetBit(j + 1)) {
TaskId targetTID = ac2TaskStates[j].getTaskId();
cInfoList.add(Pair.<TaskId, ConnectorDescriptorId>of(targetTID, cdId));
}
}
}
}
}
}
return taskConnectivity;
}
private void setLocations(FeedConnectJobInfo cInfo) {
JobSpecification jobSpec = cInfo.getSpec();
List<OperatorDescriptorId> collectOperatorIds = new ArrayList<OperatorDescriptorId>();
List<OperatorDescriptorId> computeOperatorIds = new ArrayList<OperatorDescriptorId>();
List<OperatorDescriptorId> storageOperatorIds = new ArrayList<OperatorDescriptorId>();
Map<OperatorDescriptorId, IOperatorDescriptor> operators = jobSpec.getOperatorMap();
for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) {
IOperatorDescriptor opDesc = entry.getValue();
IOperatorDescriptor actualOp = null;
if (opDesc instanceof FeedMetaOperatorDescriptor) {
actualOp = ((FeedMetaOperatorDescriptor) opDesc).getCoreOperator();
} else {
actualOp = opDesc;
}
if (actualOp instanceof AlgebricksMetaOperatorDescriptor) {
AlgebricksMetaOperatorDescriptor op = ((AlgebricksMetaOperatorDescriptor) actualOp);
IPushRuntimeFactory[] runtimeFactories = op.getPipeline().getRuntimeFactories();
boolean computeOp = false;
for (IPushRuntimeFactory rf : runtimeFactories) {
if (rf instanceof AssignRuntimeFactory) {
IConnectorDescriptor connDesc =
jobSpec.getOperatorInputMap().get(op.getOperatorId()).get(0);
IOperatorDescriptor sourceOp =
jobSpec
.getConnectorOperatorMap()
.get(connDesc.getConnectorId())
.getLeft()
.getLeft();
if (sourceOp instanceof FeedCollectOperatorDescriptor) {
computeOp = true;
break;
}
}
}
if (computeOp) {
computeOperatorIds.add(entry.getKey());
}
} else if (actualOp instanceof LSMTreeIndexInsertUpdateDeleteOperatorDescriptor) {
storageOperatorIds.add(entry.getKey());
} else if (actualOp instanceof FeedCollectOperatorDescriptor) {
collectOperatorIds.add(entry.getKey());
}
}
try {
IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
JobInfo info = hcc.getJobInfo(cInfo.getJobId());
List<String> collectLocations = new ArrayList<String>();
for (OperatorDescriptorId collectOpId : collectOperatorIds) {
Map<Integer, String> operatorLocations = info.getOperatorLocations().get(collectOpId);
int nOperatorInstances = operatorLocations.size();
for (int i = 0; i < nOperatorInstances; i++) {
collectLocations.add(operatorLocations.get(i));
}
}
List<String> computeLocations = new ArrayList<String>();
for (OperatorDescriptorId computeOpId : computeOperatorIds) {
Map<Integer, String> operatorLocations = info.getOperatorLocations().get(computeOpId);
if (operatorLocations != null) {
int nOperatorInstances = operatorLocations.size();
for (int i = 0; i < nOperatorInstances; i++) {
computeLocations.add(operatorLocations.get(i));
}
} else {
computeLocations.clear();
computeLocations.addAll(collectLocations);
}
}
List<String> storageLocations = new ArrayList<String>();
for (OperatorDescriptorId storageOpId : storageOperatorIds) {
Map<Integer, String> operatorLocations = info.getOperatorLocations().get(storageOpId);
if (operatorLocations == null) {
continue;
}
int nOperatorInstances = operatorLocations.size();
for (int i = 0; i < nOperatorInstances; i++) {
storageLocations.add(operatorLocations.get(i));
}
}
cInfo.setCollectLocations(collectLocations);
cInfo.setComputeLocations(computeLocations);
cInfo.setStorageLocations(storageLocations);
} catch (Exception e) {
e.printStackTrace();
}
}