@Test(dataProvider = "hashEnabledValues")
public void testProbeOuterJoin(
boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
TaskContext taskContext = createTaskContext();
// build
List<Type> buildTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
RowPagesBuilder buildPages =
rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT))
.addSequencePage(10, 20, 30, 40);
LookupSourceSupplier lookupSourceSupplier =
buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages);
// probe
List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.addSequencePage(15, 20, 1020, 2020).build();
OperatorFactory joinOperatorFactory =
LookupJoinOperators.probeOuterJoin(
0,
new PlanNodeId("test"),
lookupSourceSupplier,
probePages.getTypes(),
Ints.asList(0),
probePages.getHashChannel());
Operator joinOperator =
joinOperatorFactory.createOperator(
taskContext.addPipelineContext(true, true).addDriverContext());
// expected
// expected
MaterializedResult expected =
MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes))
.row("20", 1020, 2020, "20", 30, 40)
.row("21", 1021, 2021, "21", 31, 41)
.row("22", 1022, 2022, "22", 32, 42)
.row("23", 1023, 2023, "23", 33, 43)
.row("24", 1024, 2024, "24", 34, 44)
.row("25", 1025, 2025, "25", 35, 45)
.row("26", 1026, 2026, "26", 36, 46)
.row("27", 1027, 2027, "27", 37, 47)
.row("28", 1028, 2028, "28", 38, 48)
.row("29", 1029, 2029, "29", 39, 49)
.row("30", 1030, 2030, null, null, null)
.row("31", 1031, 2031, null, null, null)
.row("32", 1032, 2032, null, null, null)
.row("33", 1033, 2033, null, null, null)
.row("34", 1034, 2034, null, null, null)
.build();
assertOperatorEquals(
joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages));
}
@Test(dataProvider = "hashEnabledValues")
public void testOuterJoinWithNullOnBothSides(
boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
TaskContext taskContext = createTaskContext();
// build
RowPagesBuilder buildPages =
rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR))
.row("a")
.row((String) null)
.row((String) null)
.row("a")
.row("b");
LookupSourceSupplier lookupSourceSupplier =
buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages);
// probe
List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.row("a").row("b").row((String) null).row("c").build();
OperatorFactory joinOperatorFactory =
LookupJoinOperators.probeOuterJoin(
0,
new PlanNodeId("test"),
lookupSourceSupplier,
probePages.getTypes(),
Ints.asList(0),
probePages.getHashChannel());
Operator joinOperator =
joinOperatorFactory.createOperator(
taskContext.addPipelineContext(true, true).addDriverContext());
// expected
MaterializedResult expected =
MaterializedResult.resultBuilder(
taskContext.getSession(), concat(probeTypes, buildPages.getTypes()))
.row("a", "a")
.row("a", "a")
.row("b", "b")
.row(null, null)
.row("c", null)
.build();
assertOperatorEquals(
joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages));
}
public Session getSession() {
return taskContext.getSession();
}
private SqlTaskExecution(
TaskStateMachine taskStateMachine,
TaskContext taskContext,
SharedBuffer sharedBuffer,
PlanFragment fragment,
LocalExecutionPlanner planner,
TaskExecutor taskExecutor,
QueryMonitor queryMonitor,
Executor notificationExecutor) {
this.taskStateMachine = checkNotNull(taskStateMachine, "taskStateMachine is null");
this.taskId = taskStateMachine.getTaskId();
this.taskContext = checkNotNull(taskContext, "taskContext is null");
this.sharedBuffer = checkNotNull(sharedBuffer, "sharedBuffer is null");
this.taskExecutor = checkNotNull(taskExecutor, "driverExecutor is null");
this.notificationExecutor = checkNotNull(notificationExecutor, "notificationExecutor is null");
this.queryMonitor = checkNotNull(queryMonitor, "queryMonitor is null");
try (SetThreadName ignored = new SetThreadName("Task-%s", taskId)) {
List<DriverFactory> driverFactories;
try {
OutputFactory outputOperatorFactory;
if (fragment.getOutputPartitioning() == OutputPartitioning.NONE) {
outputOperatorFactory = new TaskOutputFactory(sharedBuffer);
} else if (fragment.getOutputPartitioning() == OutputPartitioning.HASH) {
outputOperatorFactory = new PartitionedOutputFactory(sharedBuffer);
} else {
throw new PrestoException(
NOT_SUPPORTED,
format("OutputPartitioning %s is not supported", fragment.getOutputPartitioning()));
}
LocalExecutionPlan localExecutionPlan =
planner.plan(
taskContext.getSession(),
fragment.getRoot(),
fragment.getOutputLayout(),
fragment.getSymbols(),
fragment.getDistribution(),
outputOperatorFactory);
driverFactories = localExecutionPlan.getDriverFactories();
} catch (Throwable e) {
// planning failed
taskStateMachine.failed(e);
throw Throwables.propagate(e);
}
// index driver factories
DriverSplitRunnerFactory partitionedDriverFactory = null;
ImmutableList.Builder<DriverSplitRunnerFactory> unpartitionedDriverFactories =
ImmutableList.builder();
for (DriverFactory driverFactory : driverFactories) {
if (driverFactory.getSourceIds().contains(fragment.getPartitionedSource())) {
checkState(
partitionedDriverFactory == null, "multiple partitioned sources are not supported");
partitionedDriverFactory = new DriverSplitRunnerFactory(driverFactory);
} else {
unpartitionedDriverFactories.add(new DriverSplitRunnerFactory(driverFactory));
}
}
this.unpartitionedDriverFactories = unpartitionedDriverFactories.build();
if (fragment.getDistribution() == PlanDistribution.SOURCE) {
checkArgument(
partitionedDriverFactory != null,
"Fragment is partitioned, but no partitioned driver found");
}
this.partitionedSourceId = fragment.getPartitionedSource();
this.partitionedDriverFactory = partitionedDriverFactory;
// don't register the task if it is already completed (most likely failed during planning
// above)
if (!taskStateMachine.getState().isDone()) {
taskHandle = taskExecutor.addTask(taskId);
taskStateMachine.addStateChangeListener(
new RemoveTaskHandleWhenDone(taskExecutor, taskHandle));
} else {
taskHandle = null;
}
sharedBuffer.addStateChangeListener(
new CheckTaskCompletionOnBufferFinish(SqlTaskExecution.this));
}
}