コード例 #1
0
  private void assignConnectorPolicy(ActivityCluster ac, Map<ActivityId, ActivityPlan> taskMap) {
    Map<ConnectorDescriptorId, IConnectorPolicy> cPolicyMap =
        new HashMap<ConnectorDescriptorId, IConnectorPolicy>();
    Set<ActivityId> activities = ac.getActivityMap().keySet();
    BitSet targetBitmap = new BitSet();
    for (ActivityId a1 : activities) {
      Task[] ac1TaskStates = taskMap.get(a1).getTasks();
      int nProducers = ac1TaskStates.length;
      List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(a1);
      if (outputConns != null) {
        for (IConnectorDescriptor c : outputConns) {
          ConnectorDescriptorId cdId = c.getConnectorId();
          ActivityId a2 = ac.getConsumerActivity(cdId);
          Task[] ac2TaskStates = taskMap.get(a2).getTasks();
          int nConsumers = ac2TaskStates.length;

          int[] fanouts = new int[nProducers];
          if (c.allProducersToAllConsumers()) {
            for (int i = 0; i < nProducers; ++i) {
              fanouts[i] = nConsumers;
            }
          } else {
            for (int i = 0; i < nProducers; ++i) {
              c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
              fanouts[i] = targetBitmap.cardinality();
            }
          }
          IConnectorPolicy cp = assignConnectorPolicy(ac, c, nProducers, nConsumers, fanouts);
          cPolicyMap.put(cdId, cp);
        }
      }
    }
    scheduler.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
  }
コード例 #2
0
 private Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> computeTaskConnectivity(
     JobRun jobRun, Map<ActivityId, ActivityPlan> activityPlanMap, Set<ActivityId> activities) {
   Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity =
       new HashMap<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>>();
   ActivityClusterGraph acg = jobRun.getActivityClusterGraph();
   BitSet targetBitmap = new BitSet();
   for (ActivityId ac1 : activities) {
     ActivityCluster ac = acg.getActivityMap().get(ac1);
     Task[] ac1TaskStates = activityPlanMap.get(ac1).getTasks();
     int nProducers = ac1TaskStates.length;
     List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(ac1);
     if (outputConns != null) {
       for (IConnectorDescriptor c : outputConns) {
         ConnectorDescriptorId cdId = c.getConnectorId();
         ActivityId ac2 = ac.getConsumerActivity(cdId);
         Task[] ac2TaskStates = activityPlanMap.get(ac2).getTasks();
         int nConsumers = ac2TaskStates.length;
         if (c.allProducersToAllConsumers()) {
           List<Pair<TaskId, ConnectorDescriptorId>> cInfoList =
               new ArrayList<Pair<TaskId, ConnectorDescriptorId>>();
           for (int j = 0; j < nConsumers; j++) {
             TaskId targetTID = ac2TaskStates[j].getTaskId();
             cInfoList.add(Pair.<TaskId, ConnectorDescriptorId>of(targetTID, cdId));
           }
           for (int i = 0; i < nProducers; ++i) {
             taskConnectivity.put(ac1TaskStates[i].getTaskId(), cInfoList);
           }
         } else {
           for (int i = 0; i < nProducers; ++i) {
             c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
             List<Pair<TaskId, ConnectorDescriptorId>> cInfoList =
                 taskConnectivity.get(ac1TaskStates[i].getTaskId());
             if (cInfoList == null) {
               cInfoList = new ArrayList<Pair<TaskId, ConnectorDescriptorId>>();
               taskConnectivity.put(ac1TaskStates[i].getTaskId(), cInfoList);
             }
             for (int j = targetBitmap.nextSetBit(0); j >= 0; j = targetBitmap.nextSetBit(j + 1)) {
               TaskId targetTID = ac2TaskStates[j].getTaskId();
               cInfoList.add(Pair.<TaskId, ConnectorDescriptorId>of(targetTID, cdId));
             }
           }
         }
       }
     }
   }
   return taskConnectivity;
 }
コード例 #3
0
 private Map<ActivityId, ActivityPartitionDetails> computePartitionCounts(ActivityCluster ac)
     throws HyracksException {
   PartitionConstraintSolver solver = scheduler.getSolver();
   Set<LValueConstraintExpression> lValues = new HashSet<LValueConstraintExpression>();
   for (ActivityId anId : ac.getActivityMap().keySet()) {
     lValues.add(new PartitionCountExpression(anId.getOperatorDescriptorId()));
   }
   solver.solve(lValues);
   Map<OperatorDescriptorId, Integer> nPartMap = new HashMap<OperatorDescriptorId, Integer>();
   for (LValueConstraintExpression lv : lValues) {
     Object value = solver.getValue(lv);
     if (value == null) {
       throw new HyracksException("No value found for " + lv);
     }
     if (!(value instanceof Number)) {
       throw new HyracksException(
           "Unexpected type of value bound to "
               + lv
               + ": "
               + value.getClass()
               + "("
               + value
               + ")");
     }
     int nParts = ((Number) value).intValue();
     if (nParts <= 0) {
       throw new HyracksException("Unsatisfiable number of partitions for " + lv + ": " + nParts);
     }
     nPartMap.put(
         ((PartitionCountExpression) lv).getOperatorDescriptorId(), Integer.valueOf(nParts));
   }
   Map<ActivityId, ActivityPartitionDetails> activityPartsMap =
       new HashMap<ActivityId, ActivityPartitionDetails>();
   for (ActivityId anId : ac.getActivityMap().keySet()) {
     int nParts = nPartMap.get(anId.getOperatorDescriptorId());
     int[] nInputPartitions = null;
     List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(anId);
     if (inputs != null) {
       nInputPartitions = new int[inputs.size()];
       for (int i = 0; i < nInputPartitions.length; ++i) {
         ConnectorDescriptorId cdId = inputs.get(i).getConnectorId();
         ActivityId aid = ac.getProducerActivity(cdId);
         Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId());
         nInputPartitions[i] = nPartInt;
       }
     }
     int[] nOutputPartitions = null;
     List<IConnectorDescriptor> outputs = ac.getActivityOutputMap().get(anId);
     if (outputs != null) {
       nOutputPartitions = new int[outputs.size()];
       for (int i = 0; i < nOutputPartitions.length; ++i) {
         ConnectorDescriptorId cdId = outputs.get(i).getConnectorId();
         ActivityId aid = ac.getConsumerActivity(cdId);
         Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId());
         nOutputPartitions[i] = nPartInt;
       }
     }
     ActivityPartitionDetails apd =
         new ActivityPartitionDetails(nParts, nInputPartitions, nOutputPartitions);
     activityPartsMap.put(anId, apd);
   }
   return activityPartsMap;
 }