Ejemplo n.º 1
0
 //
 // This code starts registers a callback with access to this class, and this
 // call back is access from another thread, so this code can not be placed in the constructor
 private void start() {
   // start unpartitioned drivers
   List<DriverSplitRunner> runners = new ArrayList<>();
   for (DriverSplitRunnerFactory driverFactory : unpartitionedDriverFactories) {
     runners.add(driverFactory.createDriverRunner(null));
     driverFactory.setNoMoreSplits();
   }
   enqueueDrivers(true, runners);
 }
Ejemplo n.º 2
0
 //
 // This code starts registers a callback with access to this class, and this
 // call back is access from another thread, so this code can not be placed in the constructor
 private void start() {
   // start unpartitioned drivers
   List<DriverSplitRunner> runners = new ArrayList<>();
   for (DriverSplitRunnerFactory driverFactory : unpartitionedDriverFactories) {
     for (int i = 0; i < driverFactory.getDriverInstances(); i++) {
       runners.add(driverFactory.createDriverRunner(null, false));
     }
     driverFactory.setNoMoreSplits();
   }
   enqueueDrivers(true, runners);
 }
Ejemplo n.º 3
0
  private synchronized Map<PlanNodeId, TaskSource> updateSources(List<TaskSource> sources) {
    Map<PlanNodeId, TaskSource> updatedUnpartitionedSources = new HashMap<>();

    // don't update maxAcknowledgedSplit until the end because task sources may not
    // be in sorted order and if we updated early we could skip splits
    long newMaxAcknowledgedSplit = maxAcknowledgedSplit;

    for (TaskSource source : sources) {
      PlanNodeId sourceId = source.getPlanNodeId();
      if (sourceId.equals(partitionedSourceId)) {
        // partitioned split
        ImmutableList.Builder<DriverSplitRunner> runners = ImmutableList.builder();
        for (ScheduledSplit scheduledSplit : source.getSplits()) {
          // only add a split if we have not already scheduled it
          if (scheduledSplit.getSequenceId() > maxAcknowledgedSplit) {
            // create a new driver for the split
            runners.add(partitionedDriverFactory.createDriverRunner(scheduledSplit));
            newMaxAcknowledgedSplit = max(scheduledSplit.getSequenceId(), newMaxAcknowledgedSplit);
          }
        }

        enqueueDrivers(false, runners.build());
        if (source.isNoMoreSplits()) {
          partitionedDriverFactory.setNoMoreSplits();
        }
      } else {
        // unpartitioned split

        // update newMaxAcknowledgedSplit
        for (ScheduledSplit scheduledSplit : source.getSplits()) {
          newMaxAcknowledgedSplit = max(scheduledSplit.getSequenceId(), newMaxAcknowledgedSplit);
        }

        // create new source
        TaskSource newSource;
        TaskSource currentSource = unpartitionedSources.get(sourceId);
        if (currentSource == null) {
          newSource = source;
        } else {
          newSource = currentSource.update(source);
        }

        // only record new source if something changed
        if (newSource != currentSource) {
          unpartitionedSources.put(sourceId, newSource);
          updatedUnpartitionedSources.put(sourceId, newSource);
        }
      }
    }

    maxAcknowledgedSplit = newMaxAcknowledgedSplit;
    return updatedUnpartitionedSources;
  }