private static List<Integer> getHashChannels(RowPagesBuilder probe, RowPagesBuilder build) {
ImmutableList.Builder<Integer> hashChannels = ImmutableList.builder();
if (probe.getHashChannel().isPresent()) {
hashChannels.add(probe.getHashChannel().get());
}
if (build.getHashChannel().isPresent()) {
hashChannels.add(probe.getTypes().size() + build.getHashChannel().get());
}
return hashChannels.build();
}
@Test(dataProvider = "hashEnabledValues")
public void testDistinctLimit(boolean hashEnabled) throws Exception {
RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
List<Page> input = rowPagesBuilder.addSequencePage(3, 1).addSequencePage(5, 2).build();
OperatorFactory operatorFactory =
new DistinctLimitOperator.DistinctLimitOperatorFactory(
0,
new PlanNodeId("test"),
ImmutableList.of(BIGINT),
Ints.asList(0),
5,
rowPagesBuilder.getHashChannel());
MaterializedResult expected =
resultBuilder(driverContext.getSession(), BIGINT)
.row(1L)
.row(2L)
.row(3L)
.row(4L)
.row(5L)
.build();
assertOperatorEquals(operatorFactory, driverContext, input, expected);
}
@Test(dataProvider = "hashEnabledValues")
public void testProbeOuterJoin(
boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
TaskContext taskContext = createTaskContext();
// build
List<Type> buildTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
RowPagesBuilder buildPages =
rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT))
.addSequencePage(10, 20, 30, 40);
LookupSourceSupplier lookupSourceSupplier =
buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages);
// probe
List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.addSequencePage(15, 20, 1020, 2020).build();
OperatorFactory joinOperatorFactory =
LookupJoinOperators.probeOuterJoin(
0,
new PlanNodeId("test"),
lookupSourceSupplier,
probePages.getTypes(),
Ints.asList(0),
probePages.getHashChannel());
Operator joinOperator =
joinOperatorFactory.createOperator(
taskContext.addPipelineContext(true, true).addDriverContext());
// expected
// expected
MaterializedResult expected =
MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes))
.row("20", 1020, 2020, "20", 30, 40)
.row("21", 1021, 2021, "21", 31, 41)
.row("22", 1022, 2022, "22", 32, 42)
.row("23", 1023, 2023, "23", 33, 43)
.row("24", 1024, 2024, "24", 34, 44)
.row("25", 1025, 2025, "25", 35, 45)
.row("26", 1026, 2026, "26", 36, 46)
.row("27", 1027, 2027, "27", 37, 47)
.row("28", 1028, 2028, "28", 38, 48)
.row("29", 1029, 2029, "29", 39, 49)
.row("30", 1030, 2030, null, null, null)
.row("31", 1031, 2031, null, null, null)
.row("32", 1032, 2032, null, null, null)
.row("33", 1033, 2033, null, null, null)
.row("34", 1034, 2034, null, null, null)
.build();
assertOperatorEquals(
joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages));
}
@Test(dataProvider = "hashEnabledValues")
public void testOuterJoinWithNullOnBothSides(
boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
TaskContext taskContext = createTaskContext();
// build
RowPagesBuilder buildPages =
rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR))
.row("a")
.row((String) null)
.row((String) null)
.row("a")
.row("b");
LookupSourceSupplier lookupSourceSupplier =
buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages);
// probe
List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.row("a").row("b").row((String) null).row("c").build();
OperatorFactory joinOperatorFactory =
LookupJoinOperators.probeOuterJoin(
0,
new PlanNodeId("test"),
lookupSourceSupplier,
probePages.getTypes(),
Ints.asList(0),
probePages.getHashChannel());
Operator joinOperator =
joinOperatorFactory.createOperator(
taskContext.addPipelineContext(true, true).addDriverContext());
// expected
MaterializedResult expected =
MaterializedResult.resultBuilder(
taskContext.getSession(), concat(probeTypes, buildPages.getTypes()))
.row("a", "a")
.row("a", "a")
.row("b", "b")
.row(null, null)
.row("c", null)
.build();
assertOperatorEquals(
joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages));
}
private static LookupSourceSupplier buildHash(
boolean parallelBuild,
TaskContext taskContext,
List<Integer> hashChannels,
RowPagesBuilder buildPages) {
if (parallelBuild) {
ParallelHashBuilder parallelHashBuilder =
new ParallelHashBuilder(
buildPages.getTypes(),
hashChannels,
buildPages.getHashChannel(),
100,
PARTITION_COUNT);
// collect input data
DriverContext collectDriverContext =
taskContext.addPipelineContext(true, true).addDriverContext();
ValuesOperatorFactory valuesOperatorFactory =
new ValuesOperatorFactory(
0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
OperatorFactory collectOperatorFactory =
parallelHashBuilder.getCollectOperatorFactory(1, new PlanNodeId("test"));
Driver driver =
new Driver(
collectDriverContext,
valuesOperatorFactory.createOperator(collectDriverContext),
collectOperatorFactory.createOperator(collectDriverContext));
while (!driver.isFinished()) {
driver.process();
}
// build hash tables
PipelineContext buildPipeline = taskContext.addPipelineContext(true, true);
OperatorFactory buildOperatorFactory =
parallelHashBuilder.getBuildOperatorFactory(new PlanNodeId("test"));
for (int i = 0; i < PARTITION_COUNT; i++) {
DriverContext buildDriverContext = buildPipeline.addDriverContext();
Driver buildDriver =
new Driver(buildDriverContext, buildOperatorFactory.createOperator(buildDriverContext));
while (!buildDriver.isFinished()) {
buildDriver.process();
}
}
return parallelHashBuilder.getLookupSourceSupplier();
} else {
DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext();
ValuesOperatorFactory valuesOperatorFactory =
new ValuesOperatorFactory(
0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
HashBuilderOperatorFactory hashBuilderOperatorFactory =
new HashBuilderOperatorFactory(
1,
new PlanNodeId("test"),
buildPages.getTypes(),
hashChannels,
buildPages.getHashChannel(),
100);
Driver driver =
new Driver(
driverContext,
valuesOperatorFactory.createOperator(driverContext),
hashBuilderOperatorFactory.createOperator(driverContext));
while (!driver.isFinished()) {
driver.process();
}
return hashBuilderOperatorFactory.getLookupSourceSupplier();
}
}
