private Expression createExpression(String[] arguments, Operands operands)
throws NoSuchOperatorException {
OperatorFactory factory = new OperatorFactory();
Operator operator = factory.createOperator(arguments[1]);
Expression expression = new Expression(operator, operands);
return expression;
}
@Override
public void onEntitySpawned(Entity e, Layer layer) {
if (e instanceof ControlableBody) {
log.debug("adding entity operator");
OperatorFactory<Entity, ?> fac = entOpFac.get(0);
ops.add(new OperatorBinding<Entity>(e, fac.create(e)));
}
}
@Override
public void onLayerAdded(Layer layer) {
// TODO access key for layer type and annotation
for (OperatorFactory<Layer, ?> fac : layOpFac) {
log.debug("binding layer operator");
ops.add(new OperatorBinding<Layer>(layer, fac.create(layer)));
}
}
@Test(dataProvider = "hashEnabledValues")
public void testProbeOuterJoin(
boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
TaskContext taskContext = createTaskContext();
// build
List<Type> buildTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
RowPagesBuilder buildPages =
rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT))
.addSequencePage(10, 20, 30, 40);
LookupSourceSupplier lookupSourceSupplier =
buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages);
// probe
List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.addSequencePage(15, 20, 1020, 2020).build();
OperatorFactory joinOperatorFactory =
LookupJoinOperators.probeOuterJoin(
0,
new PlanNodeId("test"),
lookupSourceSupplier,
probePages.getTypes(),
Ints.asList(0),
probePages.getHashChannel());
Operator joinOperator =
joinOperatorFactory.createOperator(
taskContext.addPipelineContext(true, true).addDriverContext());
// expected
// expected
MaterializedResult expected =
MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes))
.row("20", 1020, 2020, "20", 30, 40)
.row("21", 1021, 2021, "21", 31, 41)
.row("22", 1022, 2022, "22", 32, 42)
.row("23", 1023, 2023, "23", 33, 43)
.row("24", 1024, 2024, "24", 34, 44)
.row("25", 1025, 2025, "25", 35, 45)
.row("26", 1026, 2026, "26", 36, 46)
.row("27", 1027, 2027, "27", 37, 47)
.row("28", 1028, 2028, "28", 38, 48)
.row("29", 1029, 2029, "29", 39, 49)
.row("30", 1030, 2030, null, null, null)
.row("31", 1031, 2031, null, null, null)
.row("32", 1032, 2032, null, null, null)
.row("33", 1033, 2033, null, null, null)
.row("34", 1034, 2034, null, null, null)
.build();
assertOperatorEquals(
joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages));
}
// 20 LET x = 3
public void parse(String line) {
if (line.equalsIgnoreCase("RUN")) {
this.run();
return;
}
if (line.equalsIgnoreCase("LIST")) {
for (int l : code.keySet()) {
System.out.println(l + " " + code.get(l));
}
return;
}
try {
String parts[] = line.split(" ");
int lineNumber = Integer.parseInt(parts[0]);
String opName = parts[1];
Operator operator =
OperatorFactory.createOperator(
opName, line.substring(parts[0].length() + parts[1].length() + 2));
code.put(lineNumber, operator);
} catch (RuntimeException e) {
System.err.println("Wrong operation!");
}
}
@Test(dataProvider = "hashEnabledValues")
public void testOuterJoinWithNullOnBothSides(
boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
TaskContext taskContext = createTaskContext();
// build
RowPagesBuilder buildPages =
rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR))
.row("a")
.row((String) null)
.row((String) null)
.row("a")
.row("b");
LookupSourceSupplier lookupSourceSupplier =
buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages);
// probe
List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.row("a").row("b").row((String) null).row("c").build();
OperatorFactory joinOperatorFactory =
LookupJoinOperators.probeOuterJoin(
0,
new PlanNodeId("test"),
lookupSourceSupplier,
probePages.getTypes(),
Ints.asList(0),
probePages.getHashChannel());
Operator joinOperator =
joinOperatorFactory.createOperator(
taskContext.addPipelineContext(true, true).addDriverContext());
// expected
MaterializedResult expected =
MaterializedResult.resultBuilder(
taskContext.getSession(), concat(probeTypes, buildPages.getTypes()))
.row("a", "a")
.row("a", "a")
.row("b", "b")
.row(null, null)
.row("c", null)
.build();
assertOperatorEquals(
joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages));
}
@Test
public void testLimitWithPageAlignment() throws Exception {
List<Page> input =
rowPagesBuilder(SINGLE_LONG)
.addSequencePage(3, 1)
.addSequencePage(2, 4)
.addSequencePage(2, 6)
.build();
OperatorFactory operatorFactory =
new LimitOperatorFactory(0, ImmutableList.of(SINGLE_LONG), 5, Optional.<Integer>absent());
Operator operator = operatorFactory.createOperator(driverContext);
List<Page> expected =
rowPagesBuilder(SINGLE_LONG).addSequencePage(3, 1).addSequencePage(2, 4).build();
OperatorAssertion.assertOperatorEquals(operator, input, expected);
}
@Test
public void testAggregation() throws Exception {
List<Page> input =
rowPagesBuilder(
SINGLE_VARBINARY,
SINGLE_LONG,
SINGLE_VARBINARY,
SINGLE_LONG,
SINGLE_DOUBLE,
SINGLE_VARBINARY)
.addSequencePage(100, 0, 0, 300, 500, 500, 500)
.build();
OperatorFactory operatorFactory =
new AggregationOperatorFactory(
0,
Step.SINGLE,
ImmutableList.of(
aggregation(COUNT, new Input(0)),
aggregation(LONG_SUM, new Input(1)),
aggregation(LONG_AVERAGE, new Input(1)),
aggregation(VAR_BINARY_MAX, new Input(2)),
aggregation(COUNT_STRING_COLUMN, new Input(0)),
aggregation(LONG_SUM, new Input(3)),
aggregation(DOUBLE_SUM, new Input(4)),
aggregation(VAR_BINARY_MAX, new Input(5))));
Operator operator = operatorFactory.createOperator(driverContext);
MaterializedResult expected =
resultBuilder(
FIXED_INT_64,
FIXED_INT_64,
DOUBLE,
VARIABLE_BINARY,
FIXED_INT_64,
FIXED_INT_64,
DOUBLE,
VARIABLE_BINARY)
.row(100, 4950, 49.5, "399", 100, 54950, 54950.0, "599")
.build();
assertOperatorEquals(operator, input, expected);
}
@Test
public void testSampledLimit() throws Exception {
List<Page> input =
rowPagesBuilder(SINGLE_LONG)
.addSequencePage(2, 1)
.addSequencePage(2, 4)
.addSequencePage(2, 6)
.build();
input = appendSampleWeight(input, 2);
OperatorFactory operatorFactory =
new LimitOperatorFactory(
0, ImmutableList.of(SINGLE_LONG), 5, Optional.of(input.get(0).getChannelCount() - 1));
Operator operator = operatorFactory.createOperator(driverContext);
List<Page> expected =
rowPagesBuilder(SINGLE_LONG, SINGLE_LONG).row(1, 2).row(2, 2).pageBreak().row(4, 1).build();
OperatorAssertion.assertOperatorEquals(operator, input, expected);
}
@Test
public void test() throws Exception {
List<Page> input =
rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG).addSequencePage(100, 0, 0).build();
OperatorFactory operatorFactory =
new FilterAndProjectOperatorFactory(
0,
new FilterFunction() {
@Override
public boolean filter(TupleReadable... cursors) {
long value = cursors[1].getLong();
return 10 <= value && value < 20;
}
@Override
public boolean filter(RecordCursor cursor) {
long value = cursor.getLong(0);
return 10 <= value && value < 20;
}
},
ImmutableList.of(singleColumn(VARIABLE_BINARY, 0), new Add5Projection(1)));
Operator operator = operatorFactory.createOperator(driverContext);
MaterializedResult expected =
MaterializedResult.resultBuilder(VARIABLE_BINARY, FIXED_INT_64)
.row("10", 15)
.row("11", 16)
.row("12", 17)
.row("13", 18)
.row("14", 19)
.row("15", 20)
.row("16", 21)
.row("17", 22)
.row("18", 23)
.row("19", 24)
.build();
assertOperatorEquals(operator, input, expected);
}
private static LookupSourceSupplier buildHash(
boolean parallelBuild,
TaskContext taskContext,
List<Integer> hashChannels,
RowPagesBuilder buildPages) {
if (parallelBuild) {
ParallelHashBuilder parallelHashBuilder =
new ParallelHashBuilder(
buildPages.getTypes(),
hashChannels,
buildPages.getHashChannel(),
100,
PARTITION_COUNT);
// collect input data
DriverContext collectDriverContext =
taskContext.addPipelineContext(true, true).addDriverContext();
ValuesOperatorFactory valuesOperatorFactory =
new ValuesOperatorFactory(
0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
OperatorFactory collectOperatorFactory =
parallelHashBuilder.getCollectOperatorFactory(1, new PlanNodeId("test"));
Driver driver =
new Driver(
collectDriverContext,
valuesOperatorFactory.createOperator(collectDriverContext),
collectOperatorFactory.createOperator(collectDriverContext));
while (!driver.isFinished()) {
driver.process();
}
// build hash tables
PipelineContext buildPipeline = taskContext.addPipelineContext(true, true);
OperatorFactory buildOperatorFactory =
parallelHashBuilder.getBuildOperatorFactory(new PlanNodeId("test"));
for (int i = 0; i < PARTITION_COUNT; i++) {
DriverContext buildDriverContext = buildPipeline.addDriverContext();
Driver buildDriver =
new Driver(buildDriverContext, buildOperatorFactory.createOperator(buildDriverContext));
while (!buildDriver.isFinished()) {
buildDriver.process();
}
}
return parallelHashBuilder.getLookupSourceSupplier();
} else {
DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext();
ValuesOperatorFactory valuesOperatorFactory =
new ValuesOperatorFactory(
0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build());
HashBuilderOperatorFactory hashBuilderOperatorFactory =
new HashBuilderOperatorFactory(
1,
new PlanNodeId("test"),
buildPages.getTypes(),
hashChannels,
buildPages.getHashChannel(),
100);
Driver driver =
new Driver(
driverContext,
valuesOperatorFactory.createOperator(driverContext),
hashBuilderOperatorFactory.createOperator(driverContext));
while (!driver.isFinished()) {
driver.process();
}
return hashBuilderOperatorFactory.getLookupSourceSupplier();
}
}
