Exemplo n.º 1
0
 private DriverSplitRunner createDriverRunner(@Nullable ScheduledSplit partitionedSplit) {
   pendingCreation.incrementAndGet();
   // create driver context immediately so the driver existence is recorded in the stats
   // the number of drivers is used to balance work across nodes
   DriverContext driverContext = pipelineContext.addDriverContext();
   return new DriverSplitRunner(this, driverContext, partitionedSplit);
 }
Exemplo n.º 2
0
    private Driver createDriver(@Nullable ScheduledSplit partitionedSplit) {
      Driver driver = driverFactory.createDriver(pipelineContext.addDriverContext());

      // record driver so other threads add unpartitioned sources can see the driver
      // NOTE: this MUST be done before reading unpartitionedSources, so we see a consistent view of
      // the unpartitioned sources
      drivers.add(new WeakReference<>(driver));

      if (partitionedSplit != null) {
        // TableScanOperator requires partitioned split to be added before the first call to process
        driver.updateSource(
            new TaskSource(partitionedSourceId, ImmutableSet.of(partitionedSplit), true));
      }

      // add unpartitioned sources
      for (TaskSource source : unpartitionedSources.values()) {
        driver.updateSource(source);
      }

      pendingCreation.decrementAndGet();
      closeDriverFactoryIfFullyCreated();

      return driver;
    }
Exemplo n.º 3
0
  private static LookupSourceSupplier buildHash(
      boolean parallelBuild,
      TaskContext taskContext,
      List<Integer> hashChannels,
      RowPagesBuilder buildPages) {
    if (parallelBuild) {
      ParallelHashBuilder parallelHashBuilder =
          new ParallelHashBuilder(
              buildPages.getTypes(),
              hashChannels,
              buildPages.getHashChannel(),
              100,
              PARTITION_COUNT);

      // collect input data
      DriverContext collectDriverContext =
          taskContext.addPipelineContext(true, true).addDriverContext();
      ValuesOperatorFactory valuesOperatorFactory =
          new ValuesOperatorFactory(
              0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
      OperatorFactory collectOperatorFactory =
          parallelHashBuilder.getCollectOperatorFactory(1, new PlanNodeId("test"));
      Driver driver =
          new Driver(
              collectDriverContext,
              valuesOperatorFactory.createOperator(collectDriverContext),
              collectOperatorFactory.createOperator(collectDriverContext));

      while (!driver.isFinished()) {
        driver.process();
      }

      // build hash tables
      PipelineContext buildPipeline = taskContext.addPipelineContext(true, true);
      OperatorFactory buildOperatorFactory =
          parallelHashBuilder.getBuildOperatorFactory(new PlanNodeId("test"));
      for (int i = 0; i < PARTITION_COUNT; i++) {
        DriverContext buildDriverContext = buildPipeline.addDriverContext();
        Driver buildDriver =
            new Driver(buildDriverContext, buildOperatorFactory.createOperator(buildDriverContext));

        while (!buildDriver.isFinished()) {
          buildDriver.process();
        }
      }

      return parallelHashBuilder.getLookupSourceSupplier();
    } else {
      DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext();

      ValuesOperatorFactory valuesOperatorFactory =
          new ValuesOperatorFactory(
              0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
      HashBuilderOperatorFactory hashBuilderOperatorFactory =
          new HashBuilderOperatorFactory(
              1,
              new PlanNodeId("test"),
              buildPages.getTypes(),
              hashChannels,
              buildPages.getHashChannel(),
              100);

      Driver driver =
          new Driver(
              driverContext,
              valuesOperatorFactory.createOperator(driverContext),
              hashBuilderOperatorFactory.createOperator(driverContext));

      while (!driver.isFinished()) {
        driver.process();
      }
      return hashBuilderOperatorFactory.getLookupSourceSupplier();
    }
  }