// // This code starts registers a callback with access to this class, and this // call back is access from another thread, so this code can not be placed in the constructor private void start() { // start unpartitioned drivers List<DriverSplitRunner> runners = new ArrayList<>(); for (DriverSplitRunnerFactory driverFactory : unpartitionedDriverFactories) { runners.add(driverFactory.createDriverRunner(null)); driverFactory.setNoMoreSplits(); } enqueueDrivers(true, runners); }
// // This code starts registers a callback with access to this class, and this // call back is access from another thread, so this code can not be placed in the constructor private void start() { // start unpartitioned drivers List<DriverSplitRunner> runners = new ArrayList<>(); for (DriverSplitRunnerFactory driverFactory : unpartitionedDriverFactories) { for (int i = 0; i < driverFactory.getDriverInstances(); i++) { runners.add(driverFactory.createDriverRunner(null, false)); } driverFactory.setNoMoreSplits(); } enqueueDrivers(true, runners); }
private synchronized Map<PlanNodeId, TaskSource> updateSources(List<TaskSource> sources) { Map<PlanNodeId, TaskSource> updatedUnpartitionedSources = new HashMap<>(); // don't update maxAcknowledgedSplit until the end because task sources may not // be in sorted order and if we updated early we could skip splits long newMaxAcknowledgedSplit = maxAcknowledgedSplit; for (TaskSource source : sources) { PlanNodeId sourceId = source.getPlanNodeId(); if (sourceId.equals(partitionedSourceId)) { // partitioned split ImmutableList.Builder<DriverSplitRunner> runners = ImmutableList.builder(); for (ScheduledSplit scheduledSplit : source.getSplits()) { // only add a split if we have not already scheduled it if (scheduledSplit.getSequenceId() > maxAcknowledgedSplit) { // create a new driver for the split runners.add(partitionedDriverFactory.createDriverRunner(scheduledSplit)); newMaxAcknowledgedSplit = max(scheduledSplit.getSequenceId(), newMaxAcknowledgedSplit); } } enqueueDrivers(false, runners.build()); if (source.isNoMoreSplits()) { partitionedDriverFactory.setNoMoreSplits(); } } else { // unpartitioned split // update newMaxAcknowledgedSplit for (ScheduledSplit scheduledSplit : source.getSplits()) { newMaxAcknowledgedSplit = max(scheduledSplit.getSequenceId(), newMaxAcknowledgedSplit); } // create new source TaskSource newSource; TaskSource currentSource = unpartitionedSources.get(sourceId); if (currentSource == null) { newSource = source; } else { newSource = currentSource.update(source); } // only record new source if something changed if (newSource != currentSource) { unpartitionedSources.put(sourceId, newSource); updatedUnpartitionedSources.put(sourceId, newSource); } } } maxAcknowledgedSplit = newMaxAcknowledgedSplit; return updatedUnpartitionedSources; }