Exemplo n.º 1
0
 private static List<Integer> getHashChannels(RowPagesBuilder probe, RowPagesBuilder build) {
   ImmutableList.Builder<Integer> hashChannels = ImmutableList.builder();
   if (probe.getHashChannel().isPresent()) {
     hashChannels.add(probe.getHashChannel().get());
   }
   if (build.getHashChannel().isPresent()) {
     hashChannels.add(probe.getTypes().size() + build.getHashChannel().get());
   }
   return hashChannels.build();
 }
  @Test(dataProvider = "hashEnabledValues")
  public void testDistinctLimit(boolean hashEnabled) throws Exception {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(3, 1).addSequencePage(5, 2).build();

    OperatorFactory operatorFactory =
        new DistinctLimitOperator.DistinctLimitOperatorFactory(
            0,
            new PlanNodeId("test"),
            ImmutableList.of(BIGINT),
            Ints.asList(0),
            5,
            rowPagesBuilder.getHashChannel());

    MaterializedResult expected =
        resultBuilder(driverContext.getSession(), BIGINT)
            .row(1L)
            .row(2L)
            .row(3L)
            .row(4L)
            .row(5L)
            .build();

    assertOperatorEquals(operatorFactory, driverContext, input, expected);
  }
Exemplo n.º 3
0
  @Test(dataProvider = "hashEnabledValues")
  public void testProbeOuterJoin(
      boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
    TaskContext taskContext = createTaskContext();

    // build
    List<Type> buildTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    RowPagesBuilder buildPages =
        rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT))
            .addSequencePage(10, 20, 30, 40);
    LookupSourceSupplier lookupSourceSupplier =
        buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages);

    // probe
    List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.addSequencePage(15, 20, 1020, 2020).build();
    OperatorFactory joinOperatorFactory =
        LookupJoinOperators.probeOuterJoin(
            0,
            new PlanNodeId("test"),
            lookupSourceSupplier,
            probePages.getTypes(),
            Ints.asList(0),
            probePages.getHashChannel());
    Operator joinOperator =
        joinOperatorFactory.createOperator(
            taskContext.addPipelineContext(true, true).addDriverContext());

    // expected
    // expected
    MaterializedResult expected =
        MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes))
            .row("20", 1020, 2020, "20", 30, 40)
            .row("21", 1021, 2021, "21", 31, 41)
            .row("22", 1022, 2022, "22", 32, 42)
            .row("23", 1023, 2023, "23", 33, 43)
            .row("24", 1024, 2024, "24", 34, 44)
            .row("25", 1025, 2025, "25", 35, 45)
            .row("26", 1026, 2026, "26", 36, 46)
            .row("27", 1027, 2027, "27", 37, 47)
            .row("28", 1028, 2028, "28", 38, 48)
            .row("29", 1029, 2029, "29", 39, 49)
            .row("30", 1030, 2030, null, null, null)
            .row("31", 1031, 2031, null, null, null)
            .row("32", 1032, 2032, null, null, null)
            .row("33", 1033, 2033, null, null, null)
            .row("34", 1034, 2034, null, null, null)
            .build();

    assertOperatorEquals(
        joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages));
  }
Exemplo n.º 4
0
  @Test(dataProvider = "hashEnabledValues")
  public void testOuterJoinWithNullOnBothSides(
      boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
    TaskContext taskContext = createTaskContext();

    // build
    RowPagesBuilder buildPages =
        rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR))
            .row("a")
            .row((String) null)
            .row((String) null)
            .row("a")
            .row("b");
    LookupSourceSupplier lookupSourceSupplier =
        buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages);

    // probe
    List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row("b").row((String) null).row("c").build();
    OperatorFactory joinOperatorFactory =
        LookupJoinOperators.probeOuterJoin(
            0,
            new PlanNodeId("test"),
            lookupSourceSupplier,
            probePages.getTypes(),
            Ints.asList(0),
            probePages.getHashChannel());
    Operator joinOperator =
        joinOperatorFactory.createOperator(
            taskContext.addPipelineContext(true, true).addDriverContext());

    // expected
    MaterializedResult expected =
        MaterializedResult.resultBuilder(
                taskContext.getSession(), concat(probeTypes, buildPages.getTypes()))
            .row("a", "a")
            .row("a", "a")
            .row("b", "b")
            .row(null, null)
            .row("c", null)
            .build();

    assertOperatorEquals(
        joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages));
  }
Exemplo n.º 5
0
  private static LookupSourceSupplier buildHash(
      boolean parallelBuild,
      TaskContext taskContext,
      List<Integer> hashChannels,
      RowPagesBuilder buildPages) {
    if (parallelBuild) {
      ParallelHashBuilder parallelHashBuilder =
          new ParallelHashBuilder(
              buildPages.getTypes(),
              hashChannels,
              buildPages.getHashChannel(),
              100,
              PARTITION_COUNT);

      // collect input data
      DriverContext collectDriverContext =
          taskContext.addPipelineContext(true, true).addDriverContext();
      ValuesOperatorFactory valuesOperatorFactory =
          new ValuesOperatorFactory(
              0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
      OperatorFactory collectOperatorFactory =
          parallelHashBuilder.getCollectOperatorFactory(1, new PlanNodeId("test"));
      Driver driver =
          new Driver(
              collectDriverContext,
              valuesOperatorFactory.createOperator(collectDriverContext),
              collectOperatorFactory.createOperator(collectDriverContext));

      while (!driver.isFinished()) {
        driver.process();
      }

      // build hash tables
      PipelineContext buildPipeline = taskContext.addPipelineContext(true, true);
      OperatorFactory buildOperatorFactory =
          parallelHashBuilder.getBuildOperatorFactory(new PlanNodeId("test"));
      for (int i = 0; i < PARTITION_COUNT; i++) {
        DriverContext buildDriverContext = buildPipeline.addDriverContext();
        Driver buildDriver =
            new Driver(buildDriverContext, buildOperatorFactory.createOperator(buildDriverContext));

        while (!buildDriver.isFinished()) {
          buildDriver.process();
        }
      }

      return parallelHashBuilder.getLookupSourceSupplier();
    } else {
      DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext();

      ValuesOperatorFactory valuesOperatorFactory =
          new ValuesOperatorFactory(
              0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
      HashBuilderOperatorFactory hashBuilderOperatorFactory =
          new HashBuilderOperatorFactory(
              1,
              new PlanNodeId("test"),
              buildPages.getTypes(),
              hashChannels,
              buildPages.getHashChannel(),
              100);

      Driver driver =
          new Driver(
              driverContext,
              valuesOperatorFactory.createOperator(driverContext),
              hashBuilderOperatorFactory.createOperator(driverContext));

      while (!driver.isFinished()) {
        driver.process();
      }
      return hashBuilderOperatorFactory.getLookupSourceSupplier();
    }
  }